{"level":"info","ts":"2026-11-03T14:55:19.661Z","logger":"setup","msg":"Loaded feature gates","featureGates":{"GCSFaultToleranceEmbeddedStorage":false,"KubernetesWAS":false,"RayClusterHistoryServer":false,"RayClusterMTLS":true,"RayClusterNetworkPolicy":true,"RayClusterStatusConditions":true,"RayCronJob":false,"RayJobDeletionPolicy":true,"RayMultiHostIndexing":true,"RayServiceIncrementalUpgrade":true,"SidecarSubmitterRestart":false}}
{"level":"info","ts":"2026-11-03T14:55:19.661Z","logger":"setup","msg":"Flag watchNamespace is not set. Watch custom resources in all namespaces."}
{"level":"info","ts":"2026-11-03T14:55:19.661Z","logger":"setup","msg":"Setup manager"}
{"level":"info","ts":"2026-11-03T14:55:20.018Z","logger":"tls","msg":"Dropping unsupported cipher from TLS profile","cipher":"TLS_AES_128_GCM_SHA256"}
{"level":"info","ts":"2026-11-03T14:55:20.018Z","logger":"tls","msg":"Dropping unsupported cipher from TLS profile","cipher":"TLS_AES_256_GCM_SHA384"}
{"level":"info","ts":"2026-11-03T14:55:20.018Z","logger":"tls","msg":"Dropping unsupported cipher from TLS profile","cipher":"TLS_CHACHA20_POLY1305_SHA256"}
{"level":"info","ts":"2026-11-03T14:55:20.018Z","logger":"tls","msg":"Dropping unsupported cipher from TLS profile","cipher":"DHE-RSA-AES128-GCM-SHA256"}
{"level":"info","ts":"2026-11-03T14:55:20.018Z","logger":"tls","msg":"Dropping unsupported cipher from TLS profile","cipher":"DHE-RSA-AES256-GCM-SHA384"}
{"level":"info","ts":"2026-11-03T14:55:20.018Z","logger":"tls","msg":"Dropping unsupported cipher from TLS profile","cipher":"TLS_AES_128_GCM_SHA256"}
{"level":"info","ts":"2026-11-03T14:55:20.018Z","logger":"tls","msg":"Dropping unsupported cipher from TLS profile","cipher":"TLS_AES_256_GCM_SHA384"}
{"level":"info","ts":"2026-11-03T14:55:20.018Z","logger":"tls","msg":"Dropping unsupported cipher from TLS profile","cipher":"TLS_CHACHA20_POLY1305_SHA256"}
{"level":"info","ts":"2026-11-03T14:55:20.018Z","logger":"tls","msg":"Dropping unsupported cipher from TLS profile","cipher":"DHE-RSA-AES128-GCM-SHA256"}
{"level":"info","ts":"2026-11-03T14:55:20.018Z","logger":"tls","msg":"Dropping unsupported cipher from TLS profile","cipher":"DHE-RSA-AES256-GCM-SHA384"}
{"level":"info","ts":"2026-11-03T14:55:20.122Z","logger":"tls","msg":"TLS profile watcher registered"}
{"level":"info","ts":"2026-11-03T14:55:20.224Z","logger":"setup","msg":"RayClusterMTLS feature gate is enabled, starting mTLS controller"}
{"level":"info","ts":"2026-11-03T14:55:20.224Z","logger":"setup","msg":"NetworkPolicy controller registered"}
{"level":"info","ts":"2026-11-03T14:55:20.224Z","logger":"controller-runtime.builder","msg":"Registering a mutating webhook","GVK":"ray.io/v1, Kind=RayCluster","path":"/mutate-ray-io-v1-raycluster"}
{"level":"info","ts":"2026-11-03T14:55:20.224Z","logger":"controller-runtime.webhook","msg":"Registering webhook","path":"/mutate-ray-io-v1-raycluster"}
{"level":"info","ts":"2026-11-03T14:55:20.224Z","logger":"controller-runtime.builder","msg":"Registering a validating webhook","GVK":"ray.io/v1, Kind=RayCluster","path":"/validate-ray-io-v1-raycluster"}
{"level":"info","ts":"2026-11-03T14:55:20.224Z","logger":"controller-runtime.webhook","msg":"Registering webhook","path":"/validate-ray-io-v1-raycluster"}
{"level":"info","ts":"2026-11-03T14:55:20.224Z","logger":"controller-runtime.builder","msg":"Registering a validating webhook","GVK":"ray.io/v1, Kind=RayJob","path":"/validate-ray-io-v1-rayjob"}
{"level":"info","ts":"2026-11-03T14:55:20.224Z","logger":"controller-runtime.webhook","msg":"Registering webhook","path":"/validate-ray-io-v1-rayjob"}
{"level":"info","ts":"2026-11-03T14:55:20.224Z","logger":"controller-runtime.builder","msg":"Registering a validating webhook","GVK":"ray.io/v1, Kind=RayService","path":"/validate-ray-io-v1-rayservice"}
{"level":"info","ts":"2026-11-03T14:55:20.224Z","logger":"controller-runtime.webhook","msg":"Registering webhook","path":"/validate-ray-io-v1-rayservice"}
{"level":"info","ts":"2026-11-03T14:55:20.224Z","logger":"setup","msg":"RayCronJob feature gate is disabled, skipping RayCronJob controller setup"}
{"level":"info","ts":"2026-11-03T14:55:20.224Z","logger":"setup","msg":"starting manager","version":"v1.7.1","gitSHA":"66285e66e354b7813b8ff253689547747c914339","buildTime":"unknown"}
{"level":"info","ts":"2026-11-03T14:55:20.224Z","logger":"controller-runtime.metrics","msg":"Starting metrics server"}
{"level":"info","ts":"2026-11-03T14:55:20.224Z","msg":"starting server","name":"health probe","addr":"[::]:8082"}
{"level":"info","ts":"2026-11-03T14:55:20.224Z","logger":"controller-runtime.webhook","msg":"Starting webhook server"}
{"level":"info","ts":"2026-11-03T14:55:20.225Z","logger":"controller-runtime.certwatcher","msg":"Updated current TLS certificate","cert":"/etc/metrics-certs/tls.crt","key":"/etc/metrics-certs/tls.key"}
{"level":"info","ts":"2026-11-03T14:55:20.225Z","logger":"controller-runtime.metrics","msg":"Serving metrics server","bindAddress":":8080","secure":true}
{"level":"info","ts":"2026-11-03T14:55:20.225Z","logger":"controller-runtime.certwatcher","msg":"Updated current TLS certificate","cert":"/tmp/k8s-webhook-server/serving-certs/tls.crt","key":"/tmp/k8s-webhook-server/serving-certs/tls.key"}
{"level":"info","ts":"2026-11-03T14:55:20.225Z","logger":"controller-runtime.webhook","msg":"Serving webhook server","host":"","port":9443}
{"level":"info","ts":"2026-11-03T14:55:20.225Z","logger":"controller-runtime.certwatcher","msg":"Starting certificate poll+watcher","cert":"/etc/metrics-certs/tls.crt","key":"/etc/metrics-certs/tls.key","interval":10}
{"level":"info","ts":"2026-11-03T14:55:20.225Z","logger":"controller-runtime.certwatcher","msg":"Starting certificate poll+watcher","cert":"/tmp/k8s-webhook-server/serving-certs/tls.crt","key":"/tmp/k8s-webhook-server/serving-certs/tls.key","interval":10}
{"level":"info","ts":"2026-11-03T14:55:20.225Z","msg":"Starting EventSource","controller":"tls-profile-watcher","source":"kind source: *v1.APIServer"}
{"level":"info","ts":"2026-11-03T14:55:20.226Z","msg":"Attempting to acquire leader lease...","lock":"opendatahub/ray-operator-leader"}
{"level":"info","ts":"2026-11-03T14:55:20.329Z","msg":"Successfully acquired lease","lock":"opendatahub/ray-operator-leader"}
{"level":"info","ts":"2026-11-03T14:55:20.329Z","logger":"controllers.RayCluster","msg":"Starting EventSource","source":"kind source: *v1.PersistentVolumeClaim"}
{"level":"info","ts":"2026-11-03T14:55:20.330Z","logger":"controllers.RayService","msg":"Starting EventSource","source":"kind source: *v1.Service"}
{"level":"info","ts":"2026-11-03T14:55:20.330Z","logger":"controllers.RayCluster","msg":"Starting EventSource","source":"kind source: *v1.RayCluster"}
{"level":"info","ts":"2026-11-03T14:55:20.330Z","logger":"controllers.RayCluster","msg":"Starting EventSource","source":"kind source: *v1.Pod"}
{"level":"info","ts":"2026-11-03T14:55:20.330Z","logger":"controllers.RayCluster","msg":"Starting EventSource","source":"kind source: *v1.Service"}
{"level":"info","ts":"2026-11-03T14:55:20.330Z","logger":"controllers.RayCluster","msg":"Starting EventSource","source":"kind source: *v1.Secret"}
{"level":"info","ts":"2026-11-03T14:55:20.330Z","logger":"controllers.RayClusterMTLS","msg":"Starting EventSource","source":"kind source: *v1.RayCluster"}
{"level":"info","ts":"2026-11-03T14:55:20.330Z","logger":"controllers.NetworkPolicy","msg":"Starting EventSource","source":"kind source: *v1.NetworkPolicy"}
{"level":"info","ts":"2026-11-03T14:55:20.330Z","logger":"controllers.RayService","msg":"Starting EventSource","source":"kind source: *v1.RayService"}
{"level":"info","ts":"2026-11-03T14:55:20.330Z","logger":"controllers.RayService","msg":"Starting EventSource","source":"kind source: *v1.RayCluster"}
{"level":"info","ts":"2026-11-03T14:55:20.330Z","logger":"controllers.NetworkPolicy","msg":"Starting EventSource","source":"kind source: *v1.RayCluster"}
{"level":"info","ts":"2026-11-03T14:55:20.331Z","logger":"controllers.RayJob","msg":"Starting EventSource","source":"kind source: *v1.Job"}
{"level":"info","ts":"2026-11-03T14:55:20.331Z","logger":"controllers.RayJob","msg":"Starting EventSource","source":"kind source: *v1.RayJob"}
{"level":"info","ts":"2026-11-03T14:55:20.331Z","msg":"Starting EventSource","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","source":"kind source: *v1.Route"}
{"level":"info","ts":"2026-11-03T14:55:20.331Z","logger":"controllers.RayJob","msg":"Starting EventSource","source":"kind source: *v1.RayCluster"}
{"level":"info","ts":"2026-11-03T14:55:20.331Z","logger":"controllers.RayJob","msg":"Starting EventSource","source":"kind source: *v1.Service"}
{"level":"info","ts":"2026-11-03T14:55:20.331Z","msg":"Starting EventSource","controller":"raycluster-legacy-security","controllerGroup":"ray.io","controllerKind":"RayCluster","source":"kind source: *v1.RayCluster"}
{"level":"info","ts":"2026-11-03T14:55:20.331Z","msg":"Starting EventSource","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","source":"kind source: *v1.RayCluster"}
{"level":"info","ts":"2026-11-03T14:55:20.331Z","msg":"Starting EventSource","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","source":"kind source: *v1.ServiceAccount"}
{"level":"info","ts":"2026-11-03T14:55:20.331Z","msg":"Starting EventSource","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","source":"kind source: *v1.Service"}
{"level":"info","ts":"2026-11-03T14:55:20.458Z","msg":"Starting Controller","controller":"tls-profile-watcher"}
{"level":"info","ts":"2026-11-03T14:55:20.458Z","msg":"Starting workers","controller":"tls-profile-watcher","worker count":1}
{"level":"info","ts":"2026-11-03T14:55:20.959Z","msg":"Starting Controller","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster"}
{"level":"info","ts":"2026-11-03T14:55:20.959Z","msg":"Starting workers","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","worker count":1}
{"level":"info","ts":"2026-11-03T14:55:20.959Z","logger":"controllers.RayJob","msg":"Starting Controller"}
{"level":"info","ts":"2026-11-03T14:55:20.959Z","logger":"controllers.RayJob","msg":"Starting workers","worker count":1}
{"level":"info","ts":"2026-11-03T14:55:20.959Z","logger":"controllers.RayService","msg":"Starting Controller"}
{"level":"info","ts":"2026-11-03T14:55:20.959Z","logger":"controllers.RayService","msg":"Starting workers","worker count":1}
{"level":"info","ts":"2026-11-03T14:55:20.959Z","msg":"Starting Controller","controller":"raycluster-legacy-security","controllerGroup":"ray.io","controllerKind":"RayCluster"}
{"level":"info","ts":"2026-11-03T14:55:20.959Z","msg":"Starting workers","controller":"raycluster-legacy-security","controllerGroup":"ray.io","controllerKind":"RayCluster","worker count":1}
{"level":"info","ts":"2026-11-03T14:55:20.959Z","logger":"controllers.RayClusterMTLS","msg":"Starting Controller"}
{"level":"info","ts":"2026-11-03T14:55:20.959Z","logger":"controllers.RayClusterMTLS","msg":"Starting workers","worker count":1}
{"level":"info","ts":"2026-11-03T14:55:20.959Z","logger":"controllers.NetworkPolicy","msg":"Starting Controller"}
{"level":"info","ts":"2026-11-03T14:55:20.959Z","logger":"controllers.NetworkPolicy","msg":"Starting workers","worker count":1}
{"level":"info","ts":"2026-11-03T14:55:21.032Z","logger":"controllers.RayCluster","msg":"Starting Controller"}
{"level":"info","ts":"2026-11-03T14:55:21.032Z","logger":"controllers.RayCluster","msg":"Starting workers","worker count":1}
{"level":"info","ts":"2026-11-03T14:58:41.075Z","logger":"raycluster-resource","msg":"default","name":"raycluster"}
{"level":"info","ts":"2026-11-03T14:58:41.075Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"raycluster","namespace":"test-ns-g9m7p"}
{"level":"info","ts":"2026-11-03T14:58:41.239Z","logger":"raycluster-resource","msg":"validate create","name":"raycluster"}
{"level":"info","ts":"2026-11-03T14:58:41.293Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ca94dafb-a57e-484d-9e95-55033cf2cf0e","cluster":"raycluster"}
{"level":"info","ts":"2026-11-03T14:58:41.293Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"7a4aecdf-b0ff-488f-86e1-34fdbcc26626"}
{"level":"info","ts":"2026-11-03T14:58:41.293Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"7a4aecdf-b0ff-488f-86e1-34fdbcc26626"}
{"level":"info","ts":"2026-11-03T14:58:41.294Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"namespace":"test-ns-g9m7p","name":"raycluster","reconcileID":"318f949a-ed44-4288-97a8-494b14f15bdc","namespacedName":{"name":"raycluster","namespace":"test-ns-g9m7p"}}
{"level":"info","ts":"2026-11-03T14:58:41.294Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"namespace":"test-ns-g9m7p","name":"raycluster","reconcileID":"318f949a-ed44-4288-97a8-494b14f15bdc","mode":"IntegratedOAuth","cluster":"raycluster"}
{"level":"info","ts":"2026-11-03T14:58:41.294Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"namespace":"test-ns-g9m7p","name":"raycluster","reconcileID":"318f949a-ed44-4288-97a8-494b14f15bdc","cluster":"raycluster"}
{"level":"info","ts":"2026-11-03T14:58:41.294Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"namespace":"test-ns-g9m7p","name":"raycluster","reconcileID":"318f949a-ed44-4288-97a8-494b14f15bdc","cluster":"raycluster"}
{"level":"info","ts":"2026-11-03T14:58:41.294Z","logger":"controllers.NetworkPolicy","msg":"Reconciling NetworkPolicies for RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"84ca060a-2deb-4dab-82ee-4cdd0af95c9e","cluster":"raycluster","namespace":"test-ns-g9m7p"}
{"level":"info","ts":"2026-11-03T14:58:41.346Z","logger":"raycluster-resource","msg":"default","name":"raycluster"}
{"level":"info","ts":"2026-11-03T14:58:41.346Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"raycluster","namespace":"test-ns-g9m7p"}
{"level":"info","ts":"2026-11-03T14:58:41.356Z","logger":"controllers.NetworkPolicy","msg":"Successfully created NetworkPolicy","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"84ca060a-2deb-4dab-82ee-4cdd0af95c9e","name":"raycluster-head"}
{"level":"info","ts":"2026-11-03T14:58:41.357Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"7a4aecdf-b0ff-488f-86e1-34fdbcc26626","name":"raycluster-head-svc"}
{"level":"info","ts":"2026-11-03T14:58:41.357Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"7a4aecdf-b0ff-488f-86e1-34fdbcc26626"}
{"level":"info","ts":"2026-11-03T14:58:41.397Z","logger":"raycluster-resource","msg":"validate update","name":"raycluster"}
{"level":"info","ts":"2026-11-03T14:58:41.411Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"7a4aecdf-b0ff-488f-86e1-34fdbcc26626","error":"certificate ray-head-cert-raycluster not found: certificates.cert-manager.io \"ray-head-cert-raycluster\" not found"}
{"level":"error","ts":"2026-11-03T14:58:41.411Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"7a4aecdf-b0ff-488f-86e1-34fdbcc26626","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster not found: certificates.cert-manager.io \"ray-head-cert-raycluster\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T14:58:41.411Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"7a4aecdf-b0ff-488f-86e1-34fdbcc26626","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster not found: certificates.cert-manager.io \"ray-head-cert-raycluster\" not found"}
{"level":"info","ts":"2026-11-03T14:58:41.411Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"7a4aecdf-b0ff-488f-86e1-34fdbcc26626","filter labels":["test-ns-g9m7p",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T14:58:41.411Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"7a4aecdf-b0ff-488f-86e1-34fdbcc26626","filter labels":["test-ns-g9m7p",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T14:58:41.411Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"7a4aecdf-b0ff-488f-86e1-34fdbcc26626","filter labels":["test-ns-g9m7p",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T14:58:41.411Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"7a4aecdf-b0ff-488f-86e1-34fdbcc26626","name":"raycluster","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T14:58:41Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T14:58:41.413Z","logger":"controllers.NetworkPolicy","msg":"Successfully created NetworkPolicy","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"84ca060a-2deb-4dab-82ee-4cdd0af95c9e","name":"raycluster-workers-small-group"}
{"level":"info","ts":"2026-11-03T14:58:41.413Z","logger":"controllers.NetworkPolicy","msg":"Successfully reconciled NetworkPolicies for RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"84ca060a-2deb-4dab-82ee-4cdd0af95c9e","cluster":"raycluster","namespace":"test-ns-g9m7p"}
{"level":"info","ts":"2026-11-03T14:58:41.413Z","logger":"controllers.NetworkPolicy","msg":"Reconciling NetworkPolicies for RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ea6659b6-5c12-4caa-9ba5-93f135860d18","cluster":"raycluster","namespace":"test-ns-g9m7p"}
{"level":"info","ts":"2026-11-03T14:58:41.464Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"7a4aecdf-b0ff-488f-86e1-34fdbcc26626","name":"raycluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"raycluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-g9m7p","name":"raycluster"}}
{"level":"info","ts":"2026-11-03T14:58:41.464Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"7a4aecdf-b0ff-488f-86e1-34fdbcc26626"}
{"level":"error","ts":"2026-11-03T14:58:41.464Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"7a4aecdf-b0ff-488f-86e1-34fdbcc26626","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster not found: certificates.cert-manager.io \"ray-head-cert-raycluster\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T14:58:41.464Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"b7144d7c-a20f-4af7-baa5-e1d3c2aa3c17"}
{"level":"info","ts":"2026-11-03T14:58:41.464Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"b7144d7c-a20f-4af7-baa5-e1d3c2aa3c17"}
{"level":"info","ts":"2026-11-03T14:58:41.464Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"b7144d7c-a20f-4af7-baa5-e1d3c2aa3c17","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-11-03T14:58:41.464Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"b7144d7c-a20f-4af7-baa5-e1d3c2aa3c17"}
{"level":"info","ts":"2026-11-03T14:58:41.503Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"namespace":"test-ns-g9m7p","name":"raycluster","reconcileID":"318f949a-ed44-4288-97a8-494b14f15bdc","name":"raycluster-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-11-03T14:58:41.514Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"b7144d7c-a20f-4af7-baa5-e1d3c2aa3c17","error":"certificate ray-head-cert-raycluster not found: certificates.cert-manager.io \"ray-head-cert-raycluster\" not found"}
{"level":"error","ts":"2026-11-03T14:58:41.514Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"b7144d7c-a20f-4af7-baa5-e1d3c2aa3c17","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster not found: certificates.cert-manager.io \"ray-head-cert-raycluster\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T14:58:41.514Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"b7144d7c-a20f-4af7-baa5-e1d3c2aa3c17","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster not found: certificates.cert-manager.io \"ray-head-cert-raycluster\" not found"}
{"level":"info","ts":"2026-11-03T14:58:41.514Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"b7144d7c-a20f-4af7-baa5-e1d3c2aa3c17","filter labels":["test-ns-g9m7p",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T14:58:41.514Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"b7144d7c-a20f-4af7-baa5-e1d3c2aa3c17","filter labels":["test-ns-g9m7p",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T14:58:41.514Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"b7144d7c-a20f-4af7-baa5-e1d3c2aa3c17","filter labels":["test-ns-g9m7p",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T14:58:41.514Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"b7144d7c-a20f-4af7-baa5-e1d3c2aa3c17","name":"raycluster","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T14:58:41Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T14:58:41.523Z","logger":"controllers.NetworkPolicy","msg":"Successfully reconciled NetworkPolicies for RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ea6659b6-5c12-4caa-9ba5-93f135860d18","cluster":"raycluster","namespace":"test-ns-g9m7p"}
{"level":"info","ts":"2026-11-03T14:58:41.573Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"b7144d7c-a20f-4af7-baa5-e1d3c2aa3c17"}
{"level":"error","ts":"2026-11-03T14:58:41.573Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"b7144d7c-a20f-4af7-baa5-e1d3c2aa3c17","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster not found: certificates.cert-manager.io \"ray-head-cert-raycluster\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T14:58:41.584Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"d1f7a518-d739-40b4-9f4d-e2e49b5836f8"}
{"level":"info","ts":"2026-11-03T14:58:41.584Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"d1f7a518-d739-40b4-9f4d-e2e49b5836f8"}
{"level":"info","ts":"2026-11-03T14:58:41.584Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"d1f7a518-d739-40b4-9f4d-e2e49b5836f8","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-11-03T14:58:41.584Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"d1f7a518-d739-40b4-9f4d-e2e49b5836f8"}
{"level":"info","ts":"2026-11-03T14:58:41.633Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"d1f7a518-d739-40b4-9f4d-e2e49b5836f8","error":"certificate ray-head-cert-raycluster not found: certificates.cert-manager.io \"ray-head-cert-raycluster\" not found"}
{"level":"error","ts":"2026-11-03T14:58:41.633Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"d1f7a518-d739-40b4-9f4d-e2e49b5836f8","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster not found: certificates.cert-manager.io \"ray-head-cert-raycluster\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T14:58:41.633Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"d1f7a518-d739-40b4-9f4d-e2e49b5836f8","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster not found: certificates.cert-manager.io \"ray-head-cert-raycluster\" not found"}
{"level":"info","ts":"2026-11-03T14:58:41.633Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"d1f7a518-d739-40b4-9f4d-e2e49b5836f8","filter labels":["test-ns-g9m7p",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T14:58:41.633Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"d1f7a518-d739-40b4-9f4d-e2e49b5836f8","filter labels":["test-ns-g9m7p",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T14:58:41.633Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"d1f7a518-d739-40b4-9f4d-e2e49b5836f8","filter labels":["test-ns-g9m7p",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T14:58:41.633Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"d1f7a518-d739-40b4-9f4d-e2e49b5836f8"}
{"level":"error","ts":"2026-11-03T14:58:41.633Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"d1f7a518-d739-40b4-9f4d-e2e49b5836f8","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster not found: certificates.cert-manager.io \"ray-head-cert-raycluster\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T14:58:41.654Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"83852a67-6edf-40d3-accb-172e75b2a868"}
{"level":"info","ts":"2026-11-03T14:58:41.654Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"83852a67-6edf-40d3-accb-172e75b2a868"}
{"level":"info","ts":"2026-11-03T14:58:41.654Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"83852a67-6edf-40d3-accb-172e75b2a868","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-11-03T14:58:41.654Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"83852a67-6edf-40d3-accb-172e75b2a868"}
{"level":"info","ts":"2026-11-03T14:58:41.657Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"namespace":"test-ns-g9m7p","name":"raycluster","reconcileID":"318f949a-ed44-4288-97a8-494b14f15bdc","name":"kuberay-gateway-access","namespace":"test-ns-g9m7p","operation":"created"}
{"level":"info","ts":"2026-11-03T14:58:41.703Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"83852a67-6edf-40d3-accb-172e75b2a868","error":"certificate ray-head-cert-raycluster not found: certificates.cert-manager.io \"ray-head-cert-raycluster\" not found"}
{"level":"error","ts":"2026-11-03T14:58:41.703Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"83852a67-6edf-40d3-accb-172e75b2a868","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster not found: certificates.cert-manager.io \"ray-head-cert-raycluster\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T14:58:41.703Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"83852a67-6edf-40d3-accb-172e75b2a868","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster not found: certificates.cert-manager.io \"ray-head-cert-raycluster\" not found"}
{"level":"info","ts":"2026-11-03T14:58:41.703Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"83852a67-6edf-40d3-accb-172e75b2a868","filter labels":["test-ns-g9m7p",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T14:58:41.703Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"83852a67-6edf-40d3-accb-172e75b2a868","filter labels":["test-ns-g9m7p",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T14:58:41.703Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"83852a67-6edf-40d3-accb-172e75b2a868","filter labels":["test-ns-g9m7p",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T14:58:41.703Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"83852a67-6edf-40d3-accb-172e75b2a868"}
{"level":"error","ts":"2026-11-03T14:58:41.703Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"83852a67-6edf-40d3-accb-172e75b2a868","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster not found: certificates.cert-manager.io \"ray-head-cert-raycluster\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T14:58:41.744Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"40a052da-4cfb-4125-928d-a9478c8905c2"}
{"level":"info","ts":"2026-11-03T14:58:41.744Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"40a052da-4cfb-4125-928d-a9478c8905c2"}
{"level":"info","ts":"2026-11-03T14:58:41.744Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"40a052da-4cfb-4125-928d-a9478c8905c2","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-11-03T14:58:41.744Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"40a052da-4cfb-4125-928d-a9478c8905c2"}
{"level":"info","ts":"2026-11-03T14:58:41.797Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"40a052da-4cfb-4125-928d-a9478c8905c2","error":"certificate ray-head-cert-raycluster not found: certificates.cert-manager.io \"ray-head-cert-raycluster\" not found"}
{"level":"error","ts":"2026-11-03T14:58:41.797Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"40a052da-4cfb-4125-928d-a9478c8905c2","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster not found: certificates.cert-manager.io \"ray-head-cert-raycluster\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T14:58:41.797Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"40a052da-4cfb-4125-928d-a9478c8905c2","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster not found: certificates.cert-manager.io \"ray-head-cert-raycluster\" not found"}
{"level":"info","ts":"2026-11-03T14:58:41.797Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"40a052da-4cfb-4125-928d-a9478c8905c2","filter labels":["test-ns-g9m7p",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T14:58:41.797Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"40a052da-4cfb-4125-928d-a9478c8905c2","filter labels":["test-ns-g9m7p",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T14:58:41.797Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"40a052da-4cfb-4125-928d-a9478c8905c2","filter labels":["test-ns-g9m7p",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T14:58:41.797Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"40a052da-4cfb-4125-928d-a9478c8905c2"}
{"level":"error","ts":"2026-11-03T14:58:41.797Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"40a052da-4cfb-4125-928d-a9478c8905c2","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster not found: certificates.cert-manager.io \"ray-head-cert-raycluster\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T14:58:41.815Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"namespace":"test-ns-g9m7p","name":"raycluster","reconcileID":"318f949a-ed44-4288-97a8-494b14f15bdc","name":"test-ns-g9m7p-raycluster","operation":"created"}
{"level":"info","ts":"2026-11-03T14:58:41.878Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"26d4a694-ecb9-46a5-b8dd-c66c5c6366ce"}
{"level":"info","ts":"2026-11-03T14:58:41.878Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"26d4a694-ecb9-46a5-b8dd-c66c5c6366ce"}
{"level":"info","ts":"2026-11-03T14:58:41.878Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"26d4a694-ecb9-46a5-b8dd-c66c5c6366ce","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-11-03T14:58:41.878Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"26d4a694-ecb9-46a5-b8dd-c66c5c6366ce"}
{"level":"info","ts":"2026-11-03T14:58:42.162Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"26d4a694-ecb9-46a5-b8dd-c66c5c6366ce","error":"certificate ray-head-cert-raycluster not found: certificates.cert-manager.io \"ray-head-cert-raycluster\" not found"}
{"level":"error","ts":"2026-11-03T14:58:42.162Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"26d4a694-ecb9-46a5-b8dd-c66c5c6366ce","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster not found: certificates.cert-manager.io \"ray-head-cert-raycluster\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T14:58:42.162Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"26d4a694-ecb9-46a5-b8dd-c66c5c6366ce","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster not found: certificates.cert-manager.io \"ray-head-cert-raycluster\" not found"}
{"level":"info","ts":"2026-11-03T14:58:42.162Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"26d4a694-ecb9-46a5-b8dd-c66c5c6366ce","filter labels":["test-ns-g9m7p",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T14:58:42.162Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"26d4a694-ecb9-46a5-b8dd-c66c5c6366ce","filter labels":["test-ns-g9m7p",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T14:58:42.162Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"26d4a694-ecb9-46a5-b8dd-c66c5c6366ce","filter labels":["test-ns-g9m7p",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T14:58:42.162Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"26d4a694-ecb9-46a5-b8dd-c66c5c6366ce"}
{"level":"error","ts":"2026-11-03T14:58:42.162Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"26d4a694-ecb9-46a5-b8dd-c66c5c6366ce","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster not found: certificates.cert-manager.io \"ray-head-cert-raycluster\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T14:58:42.361Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ad427879-ff57-408e-8a8e-e01d7eabfb8d"}
{"level":"info","ts":"2026-11-03T14:58:42.361Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ad427879-ff57-408e-8a8e-e01d7eabfb8d"}
{"level":"info","ts":"2026-11-03T14:58:42.361Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ad427879-ff57-408e-8a8e-e01d7eabfb8d","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-11-03T14:58:42.361Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ad427879-ff57-408e-8a8e-e01d7eabfb8d"}
{"level":"info","ts":"2026-11-03T14:58:42.368Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"namespace":"test-ns-g9m7p","name":"raycluster","reconcileID":"318f949a-ed44-4288-97a8-494b14f15bdc","name":"kube-rbac-proxy-config-raycluster","operation":"created"}
{"level":"info","ts":"2026-11-03T14:58:42.411Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ad427879-ff57-408e-8a8e-e01d7eabfb8d","error":"certificate ray-head-cert-raycluster is not ready at current generation 1"}
{"level":"error","ts":"2026-11-03T14:58:42.411Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ad427879-ff57-408e-8a8e-e01d7eabfb8d","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster is not ready at current generation 1","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T14:58:42.411Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ad427879-ff57-408e-8a8e-e01d7eabfb8d","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster is not ready at current generation 1"}
{"level":"info","ts":"2026-11-03T14:58:42.411Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ad427879-ff57-408e-8a8e-e01d7eabfb8d","filter labels":["test-ns-g9m7p",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T14:58:42.412Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ad427879-ff57-408e-8a8e-e01d7eabfb8d","filter labels":["test-ns-g9m7p",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T14:58:42.412Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ad427879-ff57-408e-8a8e-e01d7eabfb8d","filter labels":["test-ns-g9m7p",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T14:58:42.412Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ad427879-ff57-408e-8a8e-e01d7eabfb8d"}
{"level":"error","ts":"2026-11-03T14:58:42.412Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ad427879-ff57-408e-8a8e-e01d7eabfb8d","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster is not ready at current generation 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-03T14:58:42.422Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"namespace":"test-ns-g9m7p","name":"raycluster","reconcileID":"318f949a-ed44-4288-97a8-494b14f15bdc","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"raycluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:167\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T14:58:42.422Z","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"namespace":"test-ns-g9m7p","name":"raycluster","reconcileID":"318f949a-ed44-4288-97a8-494b14f15bdc"}
{"level":"error","ts":"2026-11-03T14:58:42.422Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"namespace":"test-ns-g9m7p","name":"raycluster","reconcileID":"318f949a-ed44-4288-97a8-494b14f15bdc","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"raycluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T14:58:42.422Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"namespace":"test-ns-g9m7p","name":"raycluster","reconcileID":"9be1ae5b-9919-4f37-8e87-ac5566f17194","namespacedName":{"name":"raycluster","namespace":"test-ns-g9m7p"}}
{"level":"info","ts":"2026-11-03T14:58:42.422Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"namespace":"test-ns-g9m7p","name":"raycluster","reconcileID":"9be1ae5b-9919-4f37-8e87-ac5566f17194","mode":"IntegratedOAuth","cluster":"raycluster"}
{"level":"info","ts":"2026-11-03T14:58:42.422Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"namespace":"test-ns-g9m7p","name":"raycluster","reconcileID":"9be1ae5b-9919-4f37-8e87-ac5566f17194","cluster":"raycluster"}
{"level":"info","ts":"2026-11-03T14:58:42.484Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"namespace":"test-ns-g9m7p","name":"raycluster","reconcileID":"9be1ae5b-9919-4f37-8e87-ac5566f17194","cluster":"raycluster"}
{"level":"info","ts":"2026-11-03T14:58:42.561Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ca94dafb-a57e-484d-9e95-55033cf2cf0e","certificate":"ray-head-cert-raycluster"}
{"level":"info","ts":"2026-11-03T14:58:42.561Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ca94dafb-a57e-484d-9e95-55033cf2cf0e"}
{"level":"info","ts":"2026-11-03T14:58:42.561Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"db4c8302-99be-418c-a048-6a7cf83496c9","cluster":"raycluster"}
{"level":"info","ts":"2026-11-03T14:58:42.732Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"c51d8b4a-fed0-472b-a4d0-808b955501cb"}
{"level":"info","ts":"2026-11-03T14:58:42.732Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"c51d8b4a-fed0-472b-a4d0-808b955501cb"}
{"level":"info","ts":"2026-11-03T14:58:42.732Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"c51d8b4a-fed0-472b-a4d0-808b955501cb","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-11-03T14:58:42.732Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"c51d8b4a-fed0-472b-a4d0-808b955501cb"}
{"level":"info","ts":"2026-11-03T14:58:42.783Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"c51d8b4a-fed0-472b-a4d0-808b955501cb","error":"certificate ray-head-cert-raycluster is not ready at current generation 1"}
{"level":"error","ts":"2026-11-03T14:58:42.783Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"c51d8b4a-fed0-472b-a4d0-808b955501cb","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster is not ready at current generation 1","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T14:58:42.783Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"c51d8b4a-fed0-472b-a4d0-808b955501cb","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster is not ready at current generation 1"}
{"level":"info","ts":"2026-11-03T14:58:42.783Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"c51d8b4a-fed0-472b-a4d0-808b955501cb","filter labels":["test-ns-g9m7p",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T14:58:42.783Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"c51d8b4a-fed0-472b-a4d0-808b955501cb","filter labels":["test-ns-g9m7p",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T14:58:42.783Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"c51d8b4a-fed0-472b-a4d0-808b955501cb","filter labels":["test-ns-g9m7p",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T14:58:42.783Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"c51d8b4a-fed0-472b-a4d0-808b955501cb"}
{"level":"error","ts":"2026-11-03T14:58:42.783Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"c51d8b4a-fed0-472b-a4d0-808b955501cb","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster is not ready at current generation 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T14:58:43.012Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"db4c8302-99be-418c-a048-6a7cf83496c9","certificate":"ray-head-cert-raycluster"}
{"level":"info","ts":"2026-11-03T14:58:43.012Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"db4c8302-99be-418c-a048-6a7cf83496c9"}
{"level":"info","ts":"2026-11-03T14:58:43.424Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"6194a179-e073-4beb-8639-9b6813e8f0af"}
{"level":"info","ts":"2026-11-03T14:58:43.424Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"6194a179-e073-4beb-8639-9b6813e8f0af"}
{"level":"info","ts":"2026-11-03T14:58:43.424Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"6194a179-e073-4beb-8639-9b6813e8f0af","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-11-03T14:58:43.424Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"6194a179-e073-4beb-8639-9b6813e8f0af"}
{"level":"info","ts":"2026-11-03T14:58:43.477Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"6194a179-e073-4beb-8639-9b6813e8f0af","error":"certificate ray-head-cert-raycluster is not ready at current generation 1"}
{"level":"error","ts":"2026-11-03T14:58:43.477Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"6194a179-e073-4beb-8639-9b6813e8f0af","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster is not ready at current generation 1","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T14:58:43.477Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"6194a179-e073-4beb-8639-9b6813e8f0af","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster is not ready at current generation 1"}
{"level":"info","ts":"2026-11-03T14:58:43.477Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"6194a179-e073-4beb-8639-9b6813e8f0af","filter labels":["test-ns-g9m7p",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T14:58:43.477Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"6194a179-e073-4beb-8639-9b6813e8f0af","filter labels":["test-ns-g9m7p",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T14:58:43.477Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"6194a179-e073-4beb-8639-9b6813e8f0af","filter labels":["test-ns-g9m7p",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T14:58:43.477Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"6194a179-e073-4beb-8639-9b6813e8f0af"}
{"level":"error","ts":"2026-11-03T14:58:43.477Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"6194a179-e073-4beb-8639-9b6813e8f0af","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster is not ready at current generation 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T14:58:44.758Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"8fbfca7d-ecbb-49de-934e-d0cd32ff39fc"}
{"level":"info","ts":"2026-11-03T14:58:44.758Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"8fbfca7d-ecbb-49de-934e-d0cd32ff39fc"}
{"level":"info","ts":"2026-11-03T14:58:44.758Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"8fbfca7d-ecbb-49de-934e-d0cd32ff39fc","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-11-03T14:58:44.758Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"8fbfca7d-ecbb-49de-934e-d0cd32ff39fc"}
{"level":"info","ts":"2026-11-03T14:58:44.859Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"8fbfca7d-ecbb-49de-934e-d0cd32ff39fc","mode":"IntegratedOAuth","cluster":"raycluster"}
{"level":"info","ts":"2026-11-03T14:58:44.859Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"8fbfca7d-ecbb-49de-934e-d0cd32ff39fc","cluster":"raycluster","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-11-03T14:58:44.859Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"8fbfca7d-ecbb-49de-934e-d0cd32ff39fc","cluster":"raycluster","authType":"IntegratedOAuth","serviceAccount":"raycluster-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-11-03T14:58:44.859Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"8fbfca7d-ecbb-49de-934e-d0cd32ff39fc","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"raycluster","ray.io/group":"headgroup","ray.io/identifier":"raycluster-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-11-03T14:58:44.859Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"8fbfca7d-ecbb-49de-934e-d0cd32ff39fc","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-11-03T14:58:44.860Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"8fbfca7d-ecbb-49de-934e-d0cd32ff39fc","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-11-03T14:58:44.860Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"8fbfca7d-ecbb-49de-934e-d0cd32ff39fc","rayNodeType":"head","generatedCmd":"ulimit -n ${RAY_START_ULIMIT_OPEN_FILES:-65536}; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-11-03T14:58:44.860Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"8fbfca7d-ecbb-49de-934e-d0cd32ff39fc","enabled":true}
{"level":"info","ts":"2026-11-03T14:58:44.932Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"8fbfca7d-ecbb-49de-934e-d0cd32ff39fc","name":"raycluster-head-b4s9s"}
{"level":"info","ts":"2026-11-03T14:58:44.932Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"8fbfca7d-ecbb-49de-934e-d0cd32ff39fc","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T14:58:44.932Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"8fbfca7d-ecbb-49de-934e-d0cd32ff39fc","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T14:58:44.932Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"8fbfca7d-ecbb-49de-934e-d0cd32ff39fc","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-11-03T14:58:44.932Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"8fbfca7d-ecbb-49de-934e-d0cd32ff39fc","group":"small-group","indices":{}}
{"level":"info","ts":"2026-11-03T14:58:44.932Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"8fbfca7d-ecbb-49de-934e-d0cd32ff39fc","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-11-03T14:58:45.032Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"8fbfca7d-ecbb-49de-934e-d0cd32ff39fc","creating worker for group":"small-group","index":0,"total":1,"replicaIndex":0}
{"level":"info","ts":"2026-11-03T14:58:45.032Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"8fbfca7d-ecbb-49de-934e-d0cd32ff39fc","nodeType":"worker","rayStartParams":{"address":"raycluster-head-svc.test-ns-g9m7p.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-11-03T14:58:45.032Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"8fbfca7d-ecbb-49de-934e-d0cd32ff39fc","rayStartCmd":"ray start  --address=raycluster-head-svc.test-ns-g9m7p.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-11-03T14:58:45.032Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"8fbfca7d-ecbb-49de-934e-d0cd32ff39fc","rayNodeType":"worker","generatedCmd":"ulimit -n ${RAY_START_ULIMIT_OPEN_FILES:-65536}; ray start  --address=raycluster-head-svc.test-ns-g9m7p.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-11-03T14:58:45.032Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"8fbfca7d-ecbb-49de-934e-d0cd32ff39fc","enabled":true}
{"level":"info","ts":"2026-11-03T14:58:45.094Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"8fbfca7d-ecbb-49de-934e-d0cd32ff39fc","name":"raycluster-small-group-worker-qrhm7"}
{"level":"info","ts":"2026-11-03T14:58:45.094Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"8fbfca7d-ecbb-49de-934e-d0cd32ff39fc","podName":"raycluster-head-b4s9s","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T14:58:45.094Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"8fbfca7d-ecbb-49de-934e-d0cd32ff39fc","podName":"raycluster-head-b4s9s","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T14:58:45.094Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"8fbfca7d-ecbb-49de-934e-d0cd32ff39fc","name":"raycluster","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T14:58:41Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T14:58:42Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T14:58:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"raycluster-head-b4s9s","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T14:58:42Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T14:58:45.154Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"71131083-ce0d-4a62-9318-f9871ebb6dec"}
{"level":"info","ts":"2026-11-03T14:58:45.154Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"71131083-ce0d-4a62-9318-f9871ebb6dec"}
{"level":"info","ts":"2026-11-03T14:58:45.154Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"71131083-ce0d-4a62-9318-f9871ebb6dec","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-11-03T14:58:45.154Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"71131083-ce0d-4a62-9318-f9871ebb6dec","Found 1 head Pod":"raycluster-head-b4s9s","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T14:58:45.154Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"71131083-ce0d-4a62-9318-f9871ebb6dec","head Pod":"raycluster-head-b4s9s","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-b4s9s. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T14:58:45.154Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"8cd0a24c-d399-4885-a9a4-f80845fe8697","cluster":"raycluster"}
{"level":"info","ts":"2026-11-03T14:58:45.154Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"71131083-ce0d-4a62-9318-f9871ebb6dec","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T14:58:45.154Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"71131083-ce0d-4a62-9318-f9871ebb6dec","worker Pod":"raycluster-small-group-worker-qrhm7","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-qrhm7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T14:58:45.154Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"71131083-ce0d-4a62-9318-f9871ebb6dec","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T14:58:45.154Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"71131083-ce0d-4a62-9318-f9871ebb6dec","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T14:58:45.154Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"71131083-ce0d-4a62-9318-f9871ebb6dec","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T14:58:45.154Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"71131083-ce0d-4a62-9318-f9871ebb6dec","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T14:58:45.154Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"71131083-ce0d-4a62-9318-f9871ebb6dec","podName":"raycluster-head-b4s9s","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T14:58:45.154Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"71131083-ce0d-4a62-9318-f9871ebb6dec","podName":"raycluster-head-b4s9s","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T14:58:45.155Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"71131083-ce0d-4a62-9318-f9871ebb6dec","name":"raycluster","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T14:58:41Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T14:58:42Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T14:58:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"raycluster-head-b4s9s","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T14:58:42Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T14:58:45.209Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"71131083-ce0d-4a62-9318-f9871ebb6dec","name":"raycluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"raycluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-g9m7p","name":"raycluster"}}
{"level":"info","ts":"2026-11-03T14:58:45.209Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"71131083-ce0d-4a62-9318-f9871ebb6dec"}
{"level":"error","ts":"2026-11-03T14:58:45.209Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"71131083-ce0d-4a62-9318-f9871ebb6dec","error":"Operation cannot be fulfilled on rayclusters.ray.io \"raycluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T14:58:45.209Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"381ab9e7-177d-4274-9581-4c402911dadd"}
{"level":"info","ts":"2026-11-03T14:58:45.209Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"381ab9e7-177d-4274-9581-4c402911dadd"}
{"level":"info","ts":"2026-11-03T14:58:45.209Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"381ab9e7-177d-4274-9581-4c402911dadd","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-11-03T14:58:45.209Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"381ab9e7-177d-4274-9581-4c402911dadd","Found 1 head Pod":"raycluster-head-b4s9s","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T14:58:45.209Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"381ab9e7-177d-4274-9581-4c402911dadd","head Pod":"raycluster-head-b4s9s","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-b4s9s. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T14:58:45.209Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"381ab9e7-177d-4274-9581-4c402911dadd","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T14:58:45.209Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"381ab9e7-177d-4274-9581-4c402911dadd","worker Pod":"raycluster-small-group-worker-qrhm7","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-qrhm7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T14:58:45.209Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"381ab9e7-177d-4274-9581-4c402911dadd","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T14:58:45.209Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"381ab9e7-177d-4274-9581-4c402911dadd","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T14:58:45.209Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"381ab9e7-177d-4274-9581-4c402911dadd","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T14:58:45.209Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"381ab9e7-177d-4274-9581-4c402911dadd","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T14:58:45.209Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"381ab9e7-177d-4274-9581-4c402911dadd","podName":"raycluster-head-b4s9s","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T14:58:45.209Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"381ab9e7-177d-4274-9581-4c402911dadd","podName":"raycluster-head-b4s9s","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T14:58:45.210Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"381ab9e7-177d-4274-9581-4c402911dadd","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T14:58:45.210Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"381ab9e7-177d-4274-9581-4c402911dadd","seconds":300}
{"level":"info","ts":"2026-11-03T14:58:45.217Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"92a2b09a-e55e-424c-95ea-b18c03af207e"}
{"level":"info","ts":"2026-11-03T14:58:45.217Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"92a2b09a-e55e-424c-95ea-b18c03af207e"}
{"level":"info","ts":"2026-11-03T14:58:45.217Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"92a2b09a-e55e-424c-95ea-b18c03af207e","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-11-03T14:58:45.217Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"92a2b09a-e55e-424c-95ea-b18c03af207e","Found 1 head Pod":"raycluster-head-b4s9s","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T14:58:45.217Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"92a2b09a-e55e-424c-95ea-b18c03af207e","head Pod":"raycluster-head-b4s9s","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-b4s9s. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T14:58:45.217Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"92a2b09a-e55e-424c-95ea-b18c03af207e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T14:58:45.218Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"92a2b09a-e55e-424c-95ea-b18c03af207e","worker Pod":"raycluster-small-group-worker-qrhm7","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-qrhm7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T14:58:45.218Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"92a2b09a-e55e-424c-95ea-b18c03af207e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T14:58:45.218Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"92a2b09a-e55e-424c-95ea-b18c03af207e","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T14:58:45.218Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"92a2b09a-e55e-424c-95ea-b18c03af207e","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T14:58:45.218Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"92a2b09a-e55e-424c-95ea-b18c03af207e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T14:58:45.218Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"92a2b09a-e55e-424c-95ea-b18c03af207e","podName":"raycluster-head-b4s9s","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T14:58:45.218Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"92a2b09a-e55e-424c-95ea-b18c03af207e","podName":"raycluster-head-b4s9s","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T14:58:45.218Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"92a2b09a-e55e-424c-95ea-b18c03af207e","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T14:58:45.218Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"92a2b09a-e55e-424c-95ea-b18c03af207e","seconds":300}
{"level":"info","ts":"2026-11-03T14:58:45.677Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"7f458d6c-a76d-4d56-96cd-cf97a92d1524"}
{"level":"info","ts":"2026-11-03T14:58:45.678Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"7f458d6c-a76d-4d56-96cd-cf97a92d1524"}
{"level":"info","ts":"2026-11-03T14:58:45.678Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"7f458d6c-a76d-4d56-96cd-cf97a92d1524","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-11-03T14:58:45.678Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"7f458d6c-a76d-4d56-96cd-cf97a92d1524","Found 1 head Pod":"raycluster-head-b4s9s","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T14:58:45.678Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"7f458d6c-a76d-4d56-96cd-cf97a92d1524","head Pod":"raycluster-head-b4s9s","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-b4s9s. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T14:58:45.678Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"7f458d6c-a76d-4d56-96cd-cf97a92d1524","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T14:58:45.678Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"7f458d6c-a76d-4d56-96cd-cf97a92d1524","worker Pod":"raycluster-small-group-worker-qrhm7","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-qrhm7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T14:58:45.678Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"7f458d6c-a76d-4d56-96cd-cf97a92d1524","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T14:58:45.678Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"7f458d6c-a76d-4d56-96cd-cf97a92d1524","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T14:58:45.678Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"7f458d6c-a76d-4d56-96cd-cf97a92d1524","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T14:58:45.678Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"7f458d6c-a76d-4d56-96cd-cf97a92d1524","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T14:58:45.678Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"7f458d6c-a76d-4d56-96cd-cf97a92d1524","podName":"raycluster-head-b4s9s","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T14:58:45.678Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"7f458d6c-a76d-4d56-96cd-cf97a92d1524","podName":"raycluster-head-b4s9s","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T14:58:45.678Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"7f458d6c-a76d-4d56-96cd-cf97a92d1524","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T14:58:45.678Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"7f458d6c-a76d-4d56-96cd-cf97a92d1524","seconds":300}
{"level":"info","ts":"2026-11-03T14:58:45.837Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"05a30877-999b-43c2-bcfa-ee0a7780939d"}
{"level":"info","ts":"2026-11-03T14:58:45.837Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"05a30877-999b-43c2-bcfa-ee0a7780939d"}
{"level":"info","ts":"2026-11-03T14:58:45.837Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"05a30877-999b-43c2-bcfa-ee0a7780939d","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-11-03T14:58:45.838Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"05a30877-999b-43c2-bcfa-ee0a7780939d","Found 1 head Pod":"raycluster-head-b4s9s","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T14:58:45.838Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"05a30877-999b-43c2-bcfa-ee0a7780939d","head Pod":"raycluster-head-b4s9s","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-b4s9s. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T14:58:45.838Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"05a30877-999b-43c2-bcfa-ee0a7780939d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T14:58:45.838Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"05a30877-999b-43c2-bcfa-ee0a7780939d","worker Pod":"raycluster-small-group-worker-qrhm7","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-qrhm7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T14:58:45.838Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"05a30877-999b-43c2-bcfa-ee0a7780939d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T14:58:45.838Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"05a30877-999b-43c2-bcfa-ee0a7780939d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T14:58:45.838Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"05a30877-999b-43c2-bcfa-ee0a7780939d","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T14:58:45.838Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"05a30877-999b-43c2-bcfa-ee0a7780939d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T14:58:45.838Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"05a30877-999b-43c2-bcfa-ee0a7780939d","podName":"raycluster-head-b4s9s","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T14:58:45.838Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"05a30877-999b-43c2-bcfa-ee0a7780939d","podName":"raycluster-head-b4s9s","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T14:58:45.838Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"05a30877-999b-43c2-bcfa-ee0a7780939d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T14:58:45.838Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"05a30877-999b-43c2-bcfa-ee0a7780939d","seconds":300}
{"level":"info","ts":"2026-11-03T14:59:45.973Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ba2503a9-011c-4437-81fa-0230d380d73d","cluster":"raycluster"}
{"level":"info","ts":"2026-11-03T15:00:46.784Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"c5922910-885d-4b83-967f-463d2d08122c","cluster":"raycluster"}
{"level":"info","ts":"2026-11-03T15:00:49.103Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"510678d7-49a0-4baa-92a4-297b42a04d57"}
{"level":"info","ts":"2026-11-03T15:00:49.103Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"510678d7-49a0-4baa-92a4-297b42a04d57"}
{"level":"info","ts":"2026-11-03T15:00:49.103Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"510678d7-49a0-4baa-92a4-297b42a04d57","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:00:49.103Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"510678d7-49a0-4baa-92a4-297b42a04d57","Found 1 head Pod":"raycluster-head-b4s9s","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:00:49.103Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"510678d7-49a0-4baa-92a4-297b42a04d57","head Pod":"raycluster-head-b4s9s","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-b4s9s. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:00:49.103Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"510678d7-49a0-4baa-92a4-297b42a04d57","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:00:49.103Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"510678d7-49a0-4baa-92a4-297b42a04d57","worker Pod":"raycluster-small-group-worker-qrhm7","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-qrhm7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:00:49.103Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"510678d7-49a0-4baa-92a4-297b42a04d57","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:00:49.103Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"510678d7-49a0-4baa-92a4-297b42a04d57","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:00:49.103Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"510678d7-49a0-4baa-92a4-297b42a04d57","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:00:49.103Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"510678d7-49a0-4baa-92a4-297b42a04d57","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:00:49.103Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"510678d7-49a0-4baa-92a4-297b42a04d57","podName":"raycluster-head-b4s9s","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:00:49.103Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"510678d7-49a0-4baa-92a4-297b42a04d57","podName":"raycluster-head-b4s9s","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:00:49.103Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"510678d7-49a0-4baa-92a4-297b42a04d57","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:00:49.103Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"510678d7-49a0-4baa-92a4-297b42a04d57","seconds":300}
{"level":"info","ts":"2026-11-03T15:00:52.374Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"796c39ea-0797-46cc-a89a-2910894c571c"}
{"level":"info","ts":"2026-11-03T15:00:52.374Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"796c39ea-0797-46cc-a89a-2910894c571c"}
{"level":"info","ts":"2026-11-03T15:00:52.374Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"796c39ea-0797-46cc-a89a-2910894c571c","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:00:52.374Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"796c39ea-0797-46cc-a89a-2910894c571c","Found 1 head Pod":"raycluster-head-b4s9s","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:00:52.374Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"796c39ea-0797-46cc-a89a-2910894c571c","head Pod":"raycluster-head-b4s9s","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-b4s9s. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:00:52.374Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"796c39ea-0797-46cc-a89a-2910894c571c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:00:52.374Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"796c39ea-0797-46cc-a89a-2910894c571c","worker Pod":"raycluster-small-group-worker-qrhm7","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-qrhm7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:00:52.374Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"796c39ea-0797-46cc-a89a-2910894c571c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:00:52.374Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"796c39ea-0797-46cc-a89a-2910894c571c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:00:52.374Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"796c39ea-0797-46cc-a89a-2910894c571c","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:00:52.374Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"796c39ea-0797-46cc-a89a-2910894c571c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:00:52.374Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"796c39ea-0797-46cc-a89a-2910894c571c","podName":"raycluster-head-b4s9s","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:00:52.374Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"796c39ea-0797-46cc-a89a-2910894c571c","podName":"raycluster-head-b4s9s","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:00:52.374Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"796c39ea-0797-46cc-a89a-2910894c571c","name":"raycluster","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T14:58:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"raycluster-head-b4s9s","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T14:58:42Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:00:52Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.31","serviceIP":"10.132.0.31","podName":"raycluster-head-b4s9s","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T14:58:42Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:00:52.435Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"db4f84fd-1de6-48de-a85a-c26848fa9a09","cluster":"raycluster"}
{"level":"info","ts":"2026-11-03T15:00:52.741Z","logger":"controllers.RayClusterMTLS","msg":"Updating head certificate","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"db4f84fd-1de6-48de-a85a-c26848fa9a09","certificate":"ray-head-cert-raycluster","ipAddresses":["10.132.0.31","127.0.0.1"]}
{"level":"info","ts":"2026-11-03T15:00:53.116Z","logger":"controllers.RayClusterMTLS","msg":"Updating worker certificate","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"db4f84fd-1de6-48de-a85a-c26848fa9a09","certificate":"ray-worker-cert-raycluster","ipAddresses":["10.134.0.24","127.0.0.1"]}
{"level":"info","ts":"2026-11-03T15:00:53.373Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"db4f84fd-1de6-48de-a85a-c26848fa9a09","certificate":"ray-head-cert-raycluster"}
{"level":"info","ts":"2026-11-03T15:00:53.373Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"db4f84fd-1de6-48de-a85a-c26848fa9a09"}
{"level":"info","ts":"2026-11-03T15:00:54.435Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ca5bede3-0baf-4ad5-8a76-eddc606cf0cd"}
{"level":"info","ts":"2026-11-03T15:00:54.435Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ca5bede3-0baf-4ad5-8a76-eddc606cf0cd"}
{"level":"info","ts":"2026-11-03T15:00:54.435Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ca5bede3-0baf-4ad5-8a76-eddc606cf0cd","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:00:54.435Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ca5bede3-0baf-4ad5-8a76-eddc606cf0cd","Found 1 head Pod":"raycluster-head-b4s9s","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:00:54.435Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ca5bede3-0baf-4ad5-8a76-eddc606cf0cd","head Pod":"raycluster-head-b4s9s","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-b4s9s. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:00:54.435Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ca5bede3-0baf-4ad5-8a76-eddc606cf0cd","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:00:54.436Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ca5bede3-0baf-4ad5-8a76-eddc606cf0cd","worker Pod":"raycluster-small-group-worker-qrhm7","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-qrhm7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:00:54.436Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ca5bede3-0baf-4ad5-8a76-eddc606cf0cd","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:00:54.436Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ca5bede3-0baf-4ad5-8a76-eddc606cf0cd","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:00:54.436Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ca5bede3-0baf-4ad5-8a76-eddc606cf0cd","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:00:54.436Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ca5bede3-0baf-4ad5-8a76-eddc606cf0cd","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:00:54.436Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ca5bede3-0baf-4ad5-8a76-eddc606cf0cd","podName":"raycluster-head-b4s9s","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:00:54.436Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ca5bede3-0baf-4ad5-8a76-eddc606cf0cd","podName":"raycluster-head-b4s9s","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:00:54.436Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ca5bede3-0baf-4ad5-8a76-eddc606cf0cd","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:00:54.436Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ca5bede3-0baf-4ad5-8a76-eddc606cf0cd","seconds":300}
{"level":"info","ts":"2026-11-03T15:01:23.374Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"c23342dd-69a1-4cd3-8a86-af594d74a9b5","cluster":"raycluster"}
{"level":"info","ts":"2026-11-03T15:01:58.395Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"c7b664ed-208a-4921-bb93-7c951482e7c4"}
{"level":"info","ts":"2026-11-03T15:01:58.395Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"c7b664ed-208a-4921-bb93-7c951482e7c4"}
{"level":"info","ts":"2026-11-03T15:01:58.395Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"c7b664ed-208a-4921-bb93-7c951482e7c4","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:01:58.396Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"c7b664ed-208a-4921-bb93-7c951482e7c4","Found 1 head Pod":"raycluster-head-b4s9s","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:01:58.396Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"c7b664ed-208a-4921-bb93-7c951482e7c4","head Pod":"raycluster-head-b4s9s","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-b4s9s. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:01:58.396Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"c7b664ed-208a-4921-bb93-7c951482e7c4","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:01:58.396Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"c7b664ed-208a-4921-bb93-7c951482e7c4","worker Pod":"raycluster-small-group-worker-qrhm7","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-qrhm7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:01:58.396Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"c7b664ed-208a-4921-bb93-7c951482e7c4","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:01:58.396Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"c7b664ed-208a-4921-bb93-7c951482e7c4","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:01:58.396Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"c7b664ed-208a-4921-bb93-7c951482e7c4","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:01:58.396Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"c7b664ed-208a-4921-bb93-7c951482e7c4","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:01:58.396Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"c7b664ed-208a-4921-bb93-7c951482e7c4","podName":"raycluster-head-b4s9s","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:01:58.396Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"c7b664ed-208a-4921-bb93-7c951482e7c4","podName":"raycluster-head-b4s9s","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:01:58.396Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"c7b664ed-208a-4921-bb93-7c951482e7c4","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:01:58.396Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"c7b664ed-208a-4921-bb93-7c951482e7c4","seconds":300}
{"level":"info","ts":"2026-11-03T15:01:59.354Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"b982924b-fbb1-4b3f-934a-3c86e7b7bdf8"}
{"level":"info","ts":"2026-11-03T15:01:59.354Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"b982924b-fbb1-4b3f-934a-3c86e7b7bdf8"}
{"level":"info","ts":"2026-11-03T15:01:59.354Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"b982924b-fbb1-4b3f-934a-3c86e7b7bdf8","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:01:59.354Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"b982924b-fbb1-4b3f-934a-3c86e7b7bdf8","Found 1 head Pod":"raycluster-head-b4s9s","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:01:59.354Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"b982924b-fbb1-4b3f-934a-3c86e7b7bdf8","head Pod":"raycluster-head-b4s9s","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-b4s9s. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:01:59.354Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"b982924b-fbb1-4b3f-934a-3c86e7b7bdf8","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:01:59.354Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"b982924b-fbb1-4b3f-934a-3c86e7b7bdf8","worker Pod":"raycluster-small-group-worker-qrhm7","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-qrhm7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:01:59.354Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"b982924b-fbb1-4b3f-934a-3c86e7b7bdf8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:01:59.354Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"b982924b-fbb1-4b3f-934a-3c86e7b7bdf8","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:01:59.354Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"b982924b-fbb1-4b3f-934a-3c86e7b7bdf8","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:01:59.355Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"b982924b-fbb1-4b3f-934a-3c86e7b7bdf8","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:01:59.355Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"b982924b-fbb1-4b3f-934a-3c86e7b7bdf8","podName":"raycluster-head-b4s9s","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:01:59.355Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"b982924b-fbb1-4b3f-934a-3c86e7b7bdf8","podName":"raycluster-head-b4s9s","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:01:59.355Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"b982924b-fbb1-4b3f-934a-3c86e7b7bdf8","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:01:59.355Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"b982924b-fbb1-4b3f-934a-3c86e7b7bdf8","seconds":300}
{"level":"info","ts":"2026-11-03T15:02:17.715Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"9d4d2c6d-a972-4fad-8da1-e5d0de5de7a3"}
{"level":"info","ts":"2026-11-03T15:02:17.715Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"9d4d2c6d-a972-4fad-8da1-e5d0de5de7a3"}
{"level":"info","ts":"2026-11-03T15:02:17.715Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"9d4d2c6d-a972-4fad-8da1-e5d0de5de7a3","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:02:17.715Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"9d4d2c6d-a972-4fad-8da1-e5d0de5de7a3","Found 1 head Pod":"raycluster-head-b4s9s","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:02:17.715Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"9d4d2c6d-a972-4fad-8da1-e5d0de5de7a3","head Pod":"raycluster-head-b4s9s","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-b4s9s. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:02:17.715Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"9d4d2c6d-a972-4fad-8da1-e5d0de5de7a3","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:02:17.715Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"9d4d2c6d-a972-4fad-8da1-e5d0de5de7a3","worker Pod":"raycluster-small-group-worker-qrhm7","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-qrhm7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:02:17.715Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"9d4d2c6d-a972-4fad-8da1-e5d0de5de7a3","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:02:17.715Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"9d4d2c6d-a972-4fad-8da1-e5d0de5de7a3","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:02:17.715Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"9d4d2c6d-a972-4fad-8da1-e5d0de5de7a3","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:02:17.715Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"9d4d2c6d-a972-4fad-8da1-e5d0de5de7a3","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:02:17.715Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"9d4d2c6d-a972-4fad-8da1-e5d0de5de7a3","podName":"raycluster-head-b4s9s","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:02:17.716Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"9d4d2c6d-a972-4fad-8da1-e5d0de5de7a3","podName":"raycluster-head-b4s9s","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:02:17.716Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"9d4d2c6d-a972-4fad-8da1-e5d0de5de7a3","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:02:17.716Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"9d4d2c6d-a972-4fad-8da1-e5d0de5de7a3","seconds":300}
{"level":"info","ts":"2026-11-03T15:02:21.792Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"47626961-2010-440f-bf85-fce9e76842bd"}
{"level":"info","ts":"2026-11-03T15:02:21.792Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"47626961-2010-440f-bf85-fce9e76842bd"}
{"level":"info","ts":"2026-11-03T15:02:21.792Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"47626961-2010-440f-bf85-fce9e76842bd","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:02:21.792Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"47626961-2010-440f-bf85-fce9e76842bd","Found 1 head Pod":"raycluster-head-b4s9s","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:02:21.792Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"47626961-2010-440f-bf85-fce9e76842bd","head Pod":"raycluster-head-b4s9s","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-b4s9s. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:02:21.792Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"47626961-2010-440f-bf85-fce9e76842bd","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:02:21.792Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"47626961-2010-440f-bf85-fce9e76842bd","worker Pod":"raycluster-small-group-worker-qrhm7","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-qrhm7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:02:21.792Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"47626961-2010-440f-bf85-fce9e76842bd","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:02:21.792Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"47626961-2010-440f-bf85-fce9e76842bd","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:02:21.792Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"47626961-2010-440f-bf85-fce9e76842bd","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:02:21.792Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"47626961-2010-440f-bf85-fce9e76842bd","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:02:21.792Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"47626961-2010-440f-bf85-fce9e76842bd","podName":"raycluster-head-b4s9s","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T14:58:44Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:02:21.792Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"47626961-2010-440f-bf85-fce9e76842bd","podName":"raycluster-head-b4s9s","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T14:58:44Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:02:21.792Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"47626961-2010-440f-bf85-fce9e76842bd","name":"raycluster","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:00:52Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.31","serviceIP":"10.132.0.31","podName":"raycluster-head-b4s9s","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T14:58:42Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:02:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.31","serviceIP":"10.132.0.31","podName":"raycluster-head-b4s9s","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T14:58:42Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:02:21.853Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"743cefcb-54f3-4082-a2af-5580d36c679b","cluster":"raycluster"}
{"level":"info","ts":"2026-11-03T15:02:22.444Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ca31d143-a8ec-4aaa-b15a-f4a01b98dc12"}
{"level":"info","ts":"2026-11-03T15:02:22.444Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ca31d143-a8ec-4aaa-b15a-f4a01b98dc12"}
{"level":"info","ts":"2026-11-03T15:02:22.444Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ca31d143-a8ec-4aaa-b15a-f4a01b98dc12","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:02:22.444Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ca31d143-a8ec-4aaa-b15a-f4a01b98dc12","Found 1 head Pod":"raycluster-head-b4s9s","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:02:22.444Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ca31d143-a8ec-4aaa-b15a-f4a01b98dc12","head Pod":"raycluster-head-b4s9s","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-b4s9s. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:02:22.444Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ca31d143-a8ec-4aaa-b15a-f4a01b98dc12","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:02:22.444Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ca31d143-a8ec-4aaa-b15a-f4a01b98dc12","worker Pod":"raycluster-small-group-worker-qrhm7","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-qrhm7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:02:22.444Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ca31d143-a8ec-4aaa-b15a-f4a01b98dc12","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:02:22.444Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ca31d143-a8ec-4aaa-b15a-f4a01b98dc12","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:02:22.444Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ca31d143-a8ec-4aaa-b15a-f4a01b98dc12","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:02:22.444Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ca31d143-a8ec-4aaa-b15a-f4a01b98dc12","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:02:22.444Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ca31d143-a8ec-4aaa-b15a-f4a01b98dc12","podName":"raycluster-head-b4s9s","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T14:58:44Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:02:22.444Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ca31d143-a8ec-4aaa-b15a-f4a01b98dc12","podName":"raycluster-head-b4s9s","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T14:58:44Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:02:22.444Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ca31d143-a8ec-4aaa-b15a-f4a01b98dc12","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:02:22.444Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ca31d143-a8ec-4aaa-b15a-f4a01b98dc12","seconds":300}
{"level":"info","ts":"2026-11-03T15:02:23.451Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"0d8ee497-1ad4-4898-a71b-eff9634d5744"}
{"level":"info","ts":"2026-11-03T15:02:23.451Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"0d8ee497-1ad4-4898-a71b-eff9634d5744"}
{"level":"info","ts":"2026-11-03T15:02:23.451Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"0d8ee497-1ad4-4898-a71b-eff9634d5744","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:02:23.452Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"0d8ee497-1ad4-4898-a71b-eff9634d5744","Found 1 head Pod":"raycluster-head-b4s9s","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:02:23.452Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"0d8ee497-1ad4-4898-a71b-eff9634d5744","head Pod":"raycluster-head-b4s9s","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-b4s9s. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:02:23.452Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"0d8ee497-1ad4-4898-a71b-eff9634d5744","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:02:23.452Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"0d8ee497-1ad4-4898-a71b-eff9634d5744","worker Pod":"raycluster-small-group-worker-qrhm7","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-qrhm7. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:02:23.452Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"0d8ee497-1ad4-4898-a71b-eff9634d5744","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:02:23.452Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"0d8ee497-1ad4-4898-a71b-eff9634d5744","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:02:23.452Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"0d8ee497-1ad4-4898-a71b-eff9634d5744","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:02:23.452Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"0d8ee497-1ad4-4898-a71b-eff9634d5744","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:02:23.452Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"0d8ee497-1ad4-4898-a71b-eff9634d5744","podName":"raycluster-small-group-worker-qrhm7","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T14:58:45Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:02:23.452Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"0d8ee497-1ad4-4898-a71b-eff9634d5744","podName":"raycluster-small-group-worker-qrhm7","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T14:58:45Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:02:23.452Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"0d8ee497-1ad4-4898-a71b-eff9634d5744","name":"raycluster","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:02:21Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.31","serviceIP":"10.132.0.31","podName":"raycluster-head-b4s9s","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T14:58:42Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:02:23Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.31","serviceIP":"10.132.0.31","podName":"raycluster-head-b4s9s","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T14:58:42Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:02:23.513Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"f120e440-2c5f-4382-9acb-0f4f4f01000b","cluster":"raycluster"}
{"level":"info","ts":"2026-11-03T15:02:25.513Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"77d18534-6ca1-4183-9cc2-869633846fa6"}
{"level":"info","ts":"2026-11-03T15:02:25.513Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"77d18534-6ca1-4183-9cc2-869633846fa6"}
{"level":"info","ts":"2026-11-03T15:02:25.513Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"77d18534-6ca1-4183-9cc2-869633846fa6","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:02:25.514Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"77d18534-6ca1-4183-9cc2-869633846fa6","Found 1 head Pod":"raycluster-head-b4s9s","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:02:25.514Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"77d18534-6ca1-4183-9cc2-869633846fa6","head Pod":"raycluster-head-b4s9s","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-b4s9s. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:02:25.514Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"77d18534-6ca1-4183-9cc2-869633846fa6","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:02:25.514Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"77d18534-6ca1-4183-9cc2-869633846fa6","worker Pod":"raycluster-small-group-worker-qrhm7","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-qrhm7. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:02:25.514Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"77d18534-6ca1-4183-9cc2-869633846fa6","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:02:25.514Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"77d18534-6ca1-4183-9cc2-869633846fa6","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:02:25.514Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"77d18534-6ca1-4183-9cc2-869633846fa6","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:02:25.514Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"77d18534-6ca1-4183-9cc2-869633846fa6","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:02:25.514Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"77d18534-6ca1-4183-9cc2-869633846fa6","podName":"raycluster-head-b4s9s","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T14:58:44Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:02:25.514Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"77d18534-6ca1-4183-9cc2-869633846fa6","podName":"raycluster-head-b4s9s","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T14:58:44Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:02:25.514Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"77d18534-6ca1-4183-9cc2-869633846fa6","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:02:25.514Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"77d18534-6ca1-4183-9cc2-869633846fa6","seconds":300}
{"level":"info","ts":"2026-11-03T15:02:32.798Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"508c25b5-3d8d-4d83-afa3-ba2700ad715c"}
{"level":"info","ts":"2026-11-03T15:02:32.798Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"508c25b5-3d8d-4d83-afa3-ba2700ad715c"}
{"level":"info","ts":"2026-11-03T15:02:32.798Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"508c25b5-3d8d-4d83-afa3-ba2700ad715c","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:02:32.798Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"508c25b5-3d8d-4d83-afa3-ba2700ad715c","Found 1 head Pod":"raycluster-head-b4s9s","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:02:32.798Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"508c25b5-3d8d-4d83-afa3-ba2700ad715c","head Pod":"raycluster-head-b4s9s","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-b4s9s. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:02:32.798Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"508c25b5-3d8d-4d83-afa3-ba2700ad715c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:02:32.798Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"508c25b5-3d8d-4d83-afa3-ba2700ad715c","worker Pod":"raycluster-small-group-worker-qrhm7","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-qrhm7. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:02:32.798Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"508c25b5-3d8d-4d83-afa3-ba2700ad715c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:02:32.798Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"508c25b5-3d8d-4d83-afa3-ba2700ad715c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:02:32.798Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"508c25b5-3d8d-4d83-afa3-ba2700ad715c","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:02:32.798Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"508c25b5-3d8d-4d83-afa3-ba2700ad715c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:02:32.798Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"508c25b5-3d8d-4d83-afa3-ba2700ad715c","podName":"raycluster-small-group-worker-qrhm7","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T14:58:45Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:02:32.798Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"508c25b5-3d8d-4d83-afa3-ba2700ad715c","podName":"raycluster-small-group-worker-qrhm7","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T14:58:45Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:02:32.799Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"508c25b5-3d8d-4d83-afa3-ba2700ad715c","name":"raycluster","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:02:23Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.31","serviceIP":"10.132.0.31","podName":"raycluster-head-b4s9s","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T14:58:42Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:02:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.31","serviceIP":"10.132.0.31","podName":"raycluster-head-b4s9s","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:02:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T14:58:42Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:02:32.859Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"c4d22413-bc22-47e8-9c6e-7b686e8c9c31","cluster":"raycluster"}
{"level":"info","ts":"2026-11-03T15:02:34.469Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"c7be2328-cb77-4174-a7b4-0e9bd9e0f916"}
{"level":"info","ts":"2026-11-03T15:02:34.469Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"c7be2328-cb77-4174-a7b4-0e9bd9e0f916"}
{"level":"info","ts":"2026-11-03T15:02:34.469Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"c7be2328-cb77-4174-a7b4-0e9bd9e0f916","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:02:34.469Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"c7be2328-cb77-4174-a7b4-0e9bd9e0f916","Found 1 head Pod":"raycluster-head-b4s9s","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:02:34.469Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"c7be2328-cb77-4174-a7b4-0e9bd9e0f916","head Pod":"raycluster-head-b4s9s","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-b4s9s. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:02:34.469Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"c7be2328-cb77-4174-a7b4-0e9bd9e0f916","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:02:34.469Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"c7be2328-cb77-4174-a7b4-0e9bd9e0f916","worker Pod":"raycluster-small-group-worker-qrhm7","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-qrhm7. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:02:34.469Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"c7be2328-cb77-4174-a7b4-0e9bd9e0f916","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:02:34.469Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"c7be2328-cb77-4174-a7b4-0e9bd9e0f916","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:02:34.469Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"c7be2328-cb77-4174-a7b4-0e9bd9e0f916","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:02:34.469Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"c7be2328-cb77-4174-a7b4-0e9bd9e0f916","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:02:34.469Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"c7be2328-cb77-4174-a7b4-0e9bd9e0f916","name":"raycluster","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:02:32Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.31","serviceIP":"10.132.0.31","podName":"raycluster-head-b4s9s","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:02:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T14:58:42Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:02:34Z","stateTransitionTimes":{"ready":"2026-11-03T15:02:34Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.31","serviceIP":"10.132.0.31","podName":"raycluster-head-b4s9s","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:02:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:02:34Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T14:58:42Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:02:34.530Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ad82c5c1-4fe4-4004-a5e5-4b6794eda25d","cluster":"raycluster"}
{"level":"info","ts":"2026-11-03T15:02:35.494Z","logger":"controllers.RayJob","msg":"Skipping RayJob managed by a custom controller","RayJob":{"name":"managed-externally","namespace":"test-ns-g9m7p"},"reconcileID":"639817a8-850b-4de2-94aa-76d3992ebc3a","managed-by":"kueue.x-k8s.io/multikueue"}
{"level":"info","ts":"2026-11-03T15:02:36.531Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"e287b1a0-7aa2-4dd8-88e7-85e5f9741319"}
{"level":"info","ts":"2026-11-03T15:02:36.531Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"e287b1a0-7aa2-4dd8-88e7-85e5f9741319"}
{"level":"info","ts":"2026-11-03T15:02:36.531Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"e287b1a0-7aa2-4dd8-88e7-85e5f9741319","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:02:36.531Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"e287b1a0-7aa2-4dd8-88e7-85e5f9741319","Found 1 head Pod":"raycluster-head-b4s9s","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:02:36.531Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"e287b1a0-7aa2-4dd8-88e7-85e5f9741319","head Pod":"raycluster-head-b4s9s","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-b4s9s. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:02:36.531Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"e287b1a0-7aa2-4dd8-88e7-85e5f9741319","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:02:36.531Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"e287b1a0-7aa2-4dd8-88e7-85e5f9741319","worker Pod":"raycluster-small-group-worker-qrhm7","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-qrhm7. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:02:36.531Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"e287b1a0-7aa2-4dd8-88e7-85e5f9741319","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:02:36.531Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"e287b1a0-7aa2-4dd8-88e7-85e5f9741319","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:02:36.531Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"e287b1a0-7aa2-4dd8-88e7-85e5f9741319","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:02:36.531Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"e287b1a0-7aa2-4dd8-88e7-85e5f9741319","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:02:36.531Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"e287b1a0-7aa2-4dd8-88e7-85e5f9741319","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:02:36.531Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"e287b1a0-7aa2-4dd8-88e7-85e5f9741319","seconds":300}
{"level":"info","ts":"2026-11-03T15:02:38.535Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-g9m7p"},"reconcileID":"3bf61393-aadf-4ccc-8251-7f4f19a52d6f","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:02:38.535Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"counter","namespace":"test-ns-g9m7p"},"reconcileID":"3bf61393-aadf-4ccc-8251-7f4f19a52d6f","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-11-03T15:02:38.591Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"counter","namespace":"test-ns-g9m7p"},"reconcileID":"3bf61393-aadf-4ccc-8251-7f4f19a52d6f"}
{"level":"info","ts":"2026-11-03T15:02:38.591Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"counter","namespace":"test-ns-g9m7p"},"reconcileID":"3bf61393-aadf-4ccc-8251-7f4f19a52d6f","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-11-03T15:02:38.591Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-g9m7p"},"reconcileID":"3bf61393-aadf-4ccc-8251-7f4f19a52d6f","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-xdsdv","rayClusterName":"raycluster","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:02:38Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:02:38.591Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-g9m7p"},"reconcileID":"3bf61393-aadf-4ccc-8251-7f4f19a52d6f","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:02:38.646Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-g9m7p"},"reconcileID":"4458d876-5d75-4179-8e2d-2a3adabb61b2","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:02:38.646Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-g9m7p"},"reconcileID":"4458d876-5d75-4179-8e2d-2a3adabb61b2","name":"raycluster"}
{"level":"info","ts":"2026-11-03T15:02:38.646Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-g9m7p"},"reconcileID":"4458d876-5d75-4179-8e2d-2a3adabb61b2","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"}}
{"level":"info","ts":"2026-11-03T15:02:38.710Z","logger":"controllers.RayJob","msg":"Created submitter Kubernetes Job for RayJob","RayJob":{"name":"counter","namespace":"test-ns-g9m7p"},"reconcileID":"4458d876-5d75-4179-8e2d-2a3adabb61b2","Kubernetes Job":"counter"}
{"level":"info","ts":"2026-11-03T15:02:38.710Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-g9m7p"},"reconcileID":"4458d876-5d75-4179-8e2d-2a3adabb61b2","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-xdsdv","rayClusterName":"raycluster","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:02:38Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-xdsdv","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-g9m7p.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-11-03T15:02:38Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:02:38.710Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-g9m7p"},"reconcileID":"4458d876-5d75-4179-8e2d-2a3adabb61b2","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-11-03T15:02:38.767Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-g9m7p"},"reconcileID":"54013f47-54a9-4e08-9c8f-a5e7db7386e8","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:02:38.767Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-g9m7p"},"reconcileID":"54013f47-54a9-4e08-9c8f-a5e7db7386e8","name":"raycluster"}
{"level":"info","ts":"2026-11-03T15:02:38.767Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-g9m7p"},"reconcileID":"54013f47-54a9-4e08-9c8f-a5e7db7386e8","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"}}
{"level":"info","ts":"2026-11-03T15:02:38.778Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-g9m7p"},"reconcileID":"131ef9af-50de-4fdb-8b1b-cf59a94b942b","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:02:38.778Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-g9m7p"},"reconcileID":"131ef9af-50de-4fdb-8b1b-cf59a94b942b","name":"raycluster"}
{"level":"info","ts":"2026-11-03T15:02:38.778Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-g9m7p"},"reconcileID":"131ef9af-50de-4fdb-8b1b-cf59a94b942b","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"}}
{"level":"info","ts":"2026-11-03T15:02:41.476Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-g9m7p"},"reconcileID":"8e9e8ac5-f2a3-4605-9b2c-f52c455d87e4","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:02:41.476Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-g9m7p"},"reconcileID":"8e9e8ac5-f2a3-4605-9b2c-f52c455d87e4","name":"raycluster"}
{"level":"info","ts":"2026-11-03T15:02:41.476Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-g9m7p"},"reconcileID":"8e9e8ac5-f2a3-4605-9b2c-f52c455d87e4","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"}}
{"level":"info","ts":"2026-11-03T15:02:44.483Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-g9m7p"},"reconcileID":"0956767c-34b6-4dc9-b371-9cd1927abee9","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:02:44.483Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-g9m7p"},"reconcileID":"0956767c-34b6-4dc9-b371-9cd1927abee9","name":"raycluster"}
{"level":"info","ts":"2026-11-03T15:02:44.483Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-g9m7p"},"reconcileID":"0956767c-34b6-4dc9-b371-9cd1927abee9","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"}}
{"level":"info","ts":"2026-11-03T15:02:47.490Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-g9m7p"},"reconcileID":"32aadc7c-7cf5-485d-a353-2e1b9c45efc4","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:02:47.490Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-g9m7p"},"reconcileID":"32aadc7c-7cf5-485d-a353-2e1b9c45efc4","name":"raycluster"}
{"level":"info","ts":"2026-11-03T15:02:47.490Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-g9m7p"},"reconcileID":"32aadc7c-7cf5-485d-a353-2e1b9c45efc4","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"}}
{"level":"info","ts":"2026-11-03T15:02:47.497Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-g9m7p"},"reconcileID":"32aadc7c-7cf5-485d-a353-2e1b9c45efc4","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-xdsdv","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-g9m7p.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-11-03T15:02:38Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:02:46Z"},"jobId":"counter-xdsdv","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-g9m7p.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-11-03T15:02:38Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:02:34Z","stateTransitionTimes":{"ready":"2026-11-03T15:02:34Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.31","serviceIP":"10.132.0.31","podName":"raycluster-head-b4s9s","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:02:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:02:34Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T14:58:42Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:02:47.497Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-g9m7p"},"reconcileID":"32aadc7c-7cf5-485d-a353-2e1b9c45efc4","old JobStatus":"","new JobStatus":"PENDING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-11-03T15:02:47.593Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-g9m7p"},"reconcileID":"5b90048c-61b4-4eff-a251-5d1fda598629","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:02:47.593Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-g9m7p"},"reconcileID":"5b90048c-61b4-4eff-a251-5d1fda598629","name":"raycluster"}
{"level":"info","ts":"2026-11-03T15:02:47.593Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-g9m7p"},"reconcileID":"5b90048c-61b4-4eff-a251-5d1fda598629","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"}}
{"level":"info","ts":"2026-11-03T15:02:47.596Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-g9m7p"},"reconcileID":"5b90048c-61b4-4eff-a251-5d1fda598629","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:02:46Z"},"jobId":"counter-xdsdv","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-g9m7p.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-11-03T15:02:38Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:02:34Z","stateTransitionTimes":{"ready":"2026-11-03T15:02:34Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.31","serviceIP":"10.132.0.31","podName":"raycluster-head-b4s9s","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:02:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:02:34Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T14:58:42Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:02:46Z"},"jobId":"counter-xdsdv","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-g9m7p.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-11-03T15:02:38Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:02:34Z","stateTransitionTimes":{"ready":"2026-11-03T15:02:34Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.31","serviceIP":"10.132.0.31","podName":"raycluster-head-b4s9s","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:02:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:02:34Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T14:58:42Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:02:50.597Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-g9m7p"},"reconcileID":"30d30899-10cc-460b-92b6-e471fbfaa67f","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:02:50.597Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-g9m7p"},"reconcileID":"30d30899-10cc-460b-92b6-e471fbfaa67f","name":"raycluster"}
{"level":"info","ts":"2026-11-03T15:02:50.597Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-g9m7p"},"reconcileID":"30d30899-10cc-460b-92b6-e471fbfaa67f","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"}}
{"level":"info","ts":"2026-11-03T15:02:50.600Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-g9m7p"},"reconcileID":"30d30899-10cc-460b-92b6-e471fbfaa67f","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:02:46Z"},"jobId":"counter-xdsdv","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-g9m7p.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-11-03T15:02:38Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:02:34Z","stateTransitionTimes":{"ready":"2026-11-03T15:02:34Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.31","serviceIP":"10.132.0.31","podName":"raycluster-head-b4s9s","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:02:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:02:34Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T14:58:42Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:02:46Z"},"jobId":"counter-xdsdv","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-g9m7p.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:02:38Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:02:34Z","stateTransitionTimes":{"ready":"2026-11-03T15:02:34Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.31","serviceIP":"10.132.0.31","podName":"raycluster-head-b4s9s","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:02:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:02:34Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T14:58:42Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:02:50.600Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-g9m7p"},"reconcileID":"30d30899-10cc-460b-92b6-e471fbfaa67f","old JobStatus":"PENDING","new JobStatus":"RUNNING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-11-03T15:02:50.697Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-g9m7p"},"reconcileID":"3ed9b817-4b62-4784-abb4-984e60ff93b2","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:02:50.697Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-g9m7p"},"reconcileID":"3ed9b817-4b62-4784-abb4-984e60ff93b2","name":"raycluster"}
{"level":"info","ts":"2026-11-03T15:02:50.697Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-g9m7p"},"reconcileID":"3ed9b817-4b62-4784-abb4-984e60ff93b2","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"}}
{"level":"info","ts":"2026-11-03T15:02:50.700Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-g9m7p"},"reconcileID":"3ed9b817-4b62-4784-abb4-984e60ff93b2","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:02:46Z"},"jobId":"counter-xdsdv","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-g9m7p.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:02:38Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:02:34Z","stateTransitionTimes":{"ready":"2026-11-03T15:02:34Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.31","serviceIP":"10.132.0.31","podName":"raycluster-head-b4s9s","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:02:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:02:34Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T14:58:42Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:02:46Z"},"jobId":"counter-xdsdv","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-g9m7p.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:02:38Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:02:34Z","stateTransitionTimes":{"ready":"2026-11-03T15:02:34Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.31","serviceIP":"10.132.0.31","podName":"raycluster-head-b4s9s","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:02:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:02:34Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T14:58:42Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:02:53.701Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-g9m7p"},"reconcileID":"242b3ebf-88f4-4e07-9c22-aa9ae9c99cde","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:02:53.701Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-g9m7p"},"reconcileID":"242b3ebf-88f4-4e07-9c22-aa9ae9c99cde","name":"raycluster"}
{"level":"info","ts":"2026-11-03T15:02:53.701Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-g9m7p"},"reconcileID":"242b3ebf-88f4-4e07-9c22-aa9ae9c99cde","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"}}
{"level":"info","ts":"2026-11-03T15:02:53.704Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-g9m7p"},"reconcileID":"242b3ebf-88f4-4e07-9c22-aa9ae9c99cde","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:02:46Z"},"jobId":"counter-xdsdv","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-g9m7p.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:02:38Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:02:34Z","stateTransitionTimes":{"ready":"2026-11-03T15:02:34Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.31","serviceIP":"10.132.0.31","podName":"raycluster-head-b4s9s","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:02:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:02:34Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T14:58:42Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:02:46Z","endTime":"2026-11-03T15:02:51Z"},"jobId":"counter-xdsdv","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-g9m7p.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-11-03T15:02:38Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:02:34Z","stateTransitionTimes":{"ready":"2026-11-03T15:02:34Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.31","serviceIP":"10.132.0.31","podName":"raycluster-head-b4s9s","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:02:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:02:34Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T14:58:42Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:02:53.704Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-g9m7p"},"reconcileID":"242b3ebf-88f4-4e07-9c22-aa9ae9c99cde","old JobStatus":"RUNNING","new JobStatus":"SUCCEEDED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-11-03T15:02:53.801Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-g9m7p"},"reconcileID":"3904d4a5-4948-4522-aebb-401bd6f4d8ae","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:02:53.801Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-g9m7p"},"reconcileID":"3904d4a5-4948-4522-aebb-401bd6f4d8ae","name":"raycluster"}
{"level":"info","ts":"2026-11-03T15:02:53.801Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-g9m7p"},"reconcileID":"3904d4a5-4948-4522-aebb-401bd6f4d8ae","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"}}
{"level":"info","ts":"2026-11-03T15:02:53.804Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-g9m7p"},"reconcileID":"3904d4a5-4948-4522-aebb-401bd6f4d8ae","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:02:46Z","endTime":"2026-11-03T15:02:51Z"},"jobId":"counter-xdsdv","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-g9m7p.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-11-03T15:02:38Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:02:34Z","stateTransitionTimes":{"ready":"2026-11-03T15:02:34Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.31","serviceIP":"10.132.0.31","podName":"raycluster-head-b4s9s","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:02:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:02:34Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T14:58:42Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:02:46Z","endTime":"2026-11-03T15:02:51Z"},"jobId":"counter-xdsdv","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-g9m7p.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-11-03T15:02:38Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:02:34Z","stateTransitionTimes":{"ready":"2026-11-03T15:02:34Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.31","serviceIP":"10.132.0.31","podName":"raycluster-head-b4s9s","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:02:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:02:34Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T14:58:42Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:02:53.849Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-g9m7p"},"reconcileID":"dcf88b30-b547-4032-bb4a-ef6b634ab2d0","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:02:53.849Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"fail","namespace":"test-ns-g9m7p"},"reconcileID":"dcf88b30-b547-4032-bb4a-ef6b634ab2d0","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-11-03T15:02:53.903Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"fail","namespace":"test-ns-g9m7p"},"reconcileID":"dcf88b30-b547-4032-bb4a-ef6b634ab2d0"}
{"level":"info","ts":"2026-11-03T15:02:53.903Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"fail","namespace":"test-ns-g9m7p"},"reconcileID":"dcf88b30-b547-4032-bb4a-ef6b634ab2d0","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-11-03T15:02:53.903Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-g9m7p"},"reconcileID":"dcf88b30-b547-4032-bb4a-ef6b634ab2d0","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-bbpjq","rayClusterName":"raycluster","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:02:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:02:53.903Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-g9m7p"},"reconcileID":"dcf88b30-b547-4032-bb4a-ef6b634ab2d0","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:02:53.960Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-g9m7p"},"reconcileID":"aa33fa19-a913-4066-b8c2-f04decfc0759","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:02:53.960Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-g9m7p"},"reconcileID":"aa33fa19-a913-4066-b8c2-f04decfc0759","name":"raycluster"}
{"level":"info","ts":"2026-11-03T15:02:53.960Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-g9m7p"},"reconcileID":"aa33fa19-a913-4066-b8c2-f04decfc0759","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"}}
{"level":"info","ts":"2026-11-03T15:02:54.015Z","logger":"controllers.RayJob","msg":"Created submitter Kubernetes Job for RayJob","RayJob":{"name":"fail","namespace":"test-ns-g9m7p"},"reconcileID":"aa33fa19-a913-4066-b8c2-f04decfc0759","Kubernetes Job":"fail"}
{"level":"info","ts":"2026-11-03T15:02:54.015Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-g9m7p"},"reconcileID":"aa33fa19-a913-4066-b8c2-f04decfc0759","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-bbpjq","rayClusterName":"raycluster","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:02:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-bbpjq","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-g9m7p.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-11-03T15:02:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:02:54.015Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-g9m7p"},"reconcileID":"aa33fa19-a913-4066-b8c2-f04decfc0759","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-11-03T15:02:54.072Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-g9m7p"},"reconcileID":"9071dd71-8bef-4ff6-b6c7-00d7fe836afd","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:02:54.072Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-g9m7p"},"reconcileID":"9071dd71-8bef-4ff6-b6c7-00d7fe836afd","name":"raycluster"}
{"level":"info","ts":"2026-11-03T15:02:54.072Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-g9m7p"},"reconcileID":"9071dd71-8bef-4ff6-b6c7-00d7fe836afd","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"}}
{"level":"info","ts":"2026-11-03T15:02:56.542Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-g9m7p"},"reconcileID":"e3bc1f57-9974-4491-96c1-f0616085f284","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:02:56.542Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-g9m7p"},"reconcileID":"e3bc1f57-9974-4491-96c1-f0616085f284","name":"raycluster"}
{"level":"info","ts":"2026-11-03T15:02:56.542Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-g9m7p"},"reconcileID":"e3bc1f57-9974-4491-96c1-f0616085f284","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"}}
{"level":"info","ts":"2026-11-03T15:02:56.650Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-g9m7p"},"reconcileID":"872dbcdf-dff1-4676-8a8d-440828e146b0","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:02:56.650Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-g9m7p"},"reconcileID":"872dbcdf-dff1-4676-8a8d-440828e146b0","name":"raycluster"}
{"level":"info","ts":"2026-11-03T15:02:56.650Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-g9m7p"},"reconcileID":"872dbcdf-dff1-4676-8a8d-440828e146b0","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"}}
{"level":"info","ts":"2026-11-03T15:02:56.656Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-g9m7p"},"reconcileID":"872dbcdf-dff1-4676-8a8d-440828e146b0","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:02:46Z","endTime":"2026-11-03T15:02:51Z"},"jobId":"counter-xdsdv","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-g9m7p.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-11-03T15:02:38Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:02:34Z","stateTransitionTimes":{"ready":"2026-11-03T15:02:34Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.31","serviceIP":"10.132.0.31","podName":"raycluster-head-b4s9s","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:02:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:02:34Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T14:58:42Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:02:46Z","endTime":"2026-11-03T15:02:51Z"},"jobId":"counter-xdsdv","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-g9m7p.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-11-03T15:02:38Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:02:34Z","stateTransitionTimes":{"ready":"2026-11-03T15:02:34Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.31","serviceIP":"10.132.0.31","podName":"raycluster-head-b4s9s","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:02:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:02:34Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T14:58:42Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:02:57.658Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-g9m7p"},"reconcileID":"3cc44d0e-191c-491d-aa2c-c6bb1d8e74a9","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:02:57.658Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-g9m7p"},"reconcileID":"3cc44d0e-191c-491d-aa2c-c6bb1d8e74a9","name":"raycluster"}
{"level":"info","ts":"2026-11-03T15:02:57.658Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-g9m7p"},"reconcileID":"3cc44d0e-191c-491d-aa2c-c6bb1d8e74a9","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"}}
{"level":"info","ts":"2026-11-03T15:02:57.661Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-g9m7p"},"reconcileID":"3cc44d0e-191c-491d-aa2c-c6bb1d8e74a9","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:02:46Z","endTime":"2026-11-03T15:02:51Z"},"jobId":"counter-xdsdv","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-g9m7p.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-11-03T15:02:38Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:02:34Z","stateTransitionTimes":{"ready":"2026-11-03T15:02:34Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.31","serviceIP":"10.132.0.31","podName":"raycluster-head-b4s9s","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:02:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:02:34Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T14:58:42Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:02:46Z","endTime":"2026-11-03T15:02:51Z"},"jobId":"counter-xdsdv","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-g9m7p.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-11-03T15:02:38Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:02:34Z","stateTransitionTimes":{"ready":"2026-11-03T15:02:34Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.31","serviceIP":"10.132.0.31","podName":"raycluster-head-b4s9s","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:02:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:02:34Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T14:58:42Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:02:57.677Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-g9m7p"},"reconcileID":"18f5296b-4f46-4f03-9341-96ccf720cdb1","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:02:57.677Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-g9m7p"},"reconcileID":"18f5296b-4f46-4f03-9341-96ccf720cdb1","name":"raycluster"}
{"level":"info","ts":"2026-11-03T15:02:57.677Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-g9m7p"},"reconcileID":"18f5296b-4f46-4f03-9341-96ccf720cdb1","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"}}
{"level":"info","ts":"2026-11-03T15:02:57.680Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-g9m7p"},"reconcileID":"18f5296b-4f46-4f03-9341-96ccf720cdb1","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:02:46Z","endTime":"2026-11-03T15:02:51Z"},"jobId":"counter-xdsdv","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-g9m7p.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-11-03T15:02:38Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:02:34Z","stateTransitionTimes":{"ready":"2026-11-03T15:02:34Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.31","serviceIP":"10.132.0.31","podName":"raycluster-head-b4s9s","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:02:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:02:34Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T14:58:42Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:02:46Z","endTime":"2026-11-03T15:02:51Z"},"jobId":"counter-xdsdv","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-g9m7p.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Complete","message":"Job finished successfully.","startTime":"2026-11-03T15:02:38Z","succeeded":1,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:02:34Z","stateTransitionTimes":{"ready":"2026-11-03T15:02:34Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.31","serviceIP":"10.132.0.31","podName":"raycluster-head-b4s9s","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:02:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:02:34Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T14:58:42Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:02:57.680Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-g9m7p"},"reconcileID":"18f5296b-4f46-4f03-9341-96ccf720cdb1","old JobStatus":"SUCCEEDED","new JobStatus":"SUCCEEDED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Complete"}
{"level":"info","ts":"2026-11-03T15:02:57.739Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-g9m7p"},"reconcileID":"80accae4-4563-4dde-9d26-0d39a76ca459","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:02:57.739Z","logger":"controllers.RayJob","msg":"RayJob is using an existing RayCluster via clusterSelector; skipping resource deletion.","RayJob":{"name":"counter","namespace":"test-ns-g9m7p"},"reconcileID":"80accae4-4563-4dde-9d26-0d39a76ca459","RayClusterSelector":{"ray.io/cluster":"raycluster"}}
{"level":"info","ts":"2026-11-03T15:02:59.549Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-g9m7p"},"reconcileID":"255cf6b4-bf66-4ab4-9a14-615ec269d025","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:02:59.549Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-g9m7p"},"reconcileID":"255cf6b4-bf66-4ab4-9a14-615ec269d025","name":"raycluster"}
{"level":"info","ts":"2026-11-03T15:02:59.549Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-g9m7p"},"reconcileID":"255cf6b4-bf66-4ab4-9a14-615ec269d025","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"}}
{"level":"info","ts":"2026-11-03T15:03:02.553Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-g9m7p"},"reconcileID":"c7cfa166-817f-4cde-8c64-da421fa96d81","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:03:02.553Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-g9m7p"},"reconcileID":"c7cfa166-817f-4cde-8c64-da421fa96d81","name":"raycluster"}
{"level":"info","ts":"2026-11-03T15:03:02.553Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-g9m7p"},"reconcileID":"c7cfa166-817f-4cde-8c64-da421fa96d81","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"}}
{"level":"info","ts":"2026-11-03T15:03:02.559Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-g9m7p"},"reconcileID":"c7cfa166-817f-4cde-8c64-da421fa96d81","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-bbpjq","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-g9m7p.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-11-03T15:02:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:03:01Z"},"jobId":"fail-bbpjq","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-g9m7p.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-11-03T15:02:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:02:34Z","stateTransitionTimes":{"ready":"2026-11-03T15:02:34Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.31","serviceIP":"10.132.0.31","podName":"raycluster-head-b4s9s","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:02:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:02:34Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T14:58:42Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:03:02.559Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-g9m7p"},"reconcileID":"c7cfa166-817f-4cde-8c64-da421fa96d81","old JobStatus":"","new JobStatus":"PENDING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-11-03T15:03:02.656Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-g9m7p"},"reconcileID":"d975ed1e-45fc-47dd-9cc6-d84dbbd5bbe1","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:03:02.656Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-g9m7p"},"reconcileID":"d975ed1e-45fc-47dd-9cc6-d84dbbd5bbe1","name":"raycluster"}
{"level":"info","ts":"2026-11-03T15:03:02.656Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-g9m7p"},"reconcileID":"d975ed1e-45fc-47dd-9cc6-d84dbbd5bbe1","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"}}
{"level":"info","ts":"2026-11-03T15:03:02.659Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-g9m7p"},"reconcileID":"d975ed1e-45fc-47dd-9cc6-d84dbbd5bbe1","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:03:01Z"},"jobId":"fail-bbpjq","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-g9m7p.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-11-03T15:02:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:02:34Z","stateTransitionTimes":{"ready":"2026-11-03T15:02:34Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.31","serviceIP":"10.132.0.31","podName":"raycluster-head-b4s9s","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:02:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:02:34Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T14:58:42Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:03:01Z"},"jobId":"fail-bbpjq","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-g9m7p.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-11-03T15:02:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:02:34Z","stateTransitionTimes":{"ready":"2026-11-03T15:02:34Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.31","serviceIP":"10.132.0.31","podName":"raycluster-head-b4s9s","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:02:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:02:34Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T14:58:42Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:03:05.660Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-g9m7p"},"reconcileID":"9b2e0fb4-13af-4980-bd45-9cf53627a337","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:03:05.660Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-g9m7p"},"reconcileID":"9b2e0fb4-13af-4980-bd45-9cf53627a337","name":"raycluster"}
{"level":"info","ts":"2026-11-03T15:03:05.660Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-g9m7p"},"reconcileID":"9b2e0fb4-13af-4980-bd45-9cf53627a337","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"}}
{"level":"info","ts":"2026-11-03T15:03:05.663Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-g9m7p"},"reconcileID":"9b2e0fb4-13af-4980-bd45-9cf53627a337","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:03:01Z"},"jobId":"fail-bbpjq","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-g9m7p.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-11-03T15:02:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:02:34Z","stateTransitionTimes":{"ready":"2026-11-03T15:02:34Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.31","serviceIP":"10.132.0.31","podName":"raycluster-head-b4s9s","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:02:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:02:34Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T14:58:42Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:03:01Z","endTime":"2026-11-03T15:03:03Z"},"jobId":"fail-bbpjq","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-g9m7p.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-11-03 15:03:01,918\tINFO job_manager.py:583 -- Runtime env is setting up.\nRunning entrypoint for job fail-bbpjq: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-11-03T15:02:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:02:34Z","stateTransitionTimes":{"ready":"2026-11-03T15:02:34Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.31","serviceIP":"10.132.0.31","podName":"raycluster-head-b4s9s","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:02:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:02:34Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T14:58:42Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:03:05.663Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-g9m7p"},"reconcileID":"9b2e0fb4-13af-4980-bd45-9cf53627a337","old JobStatus":"PENDING","new JobStatus":"FAILED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-11-03T15:03:05.769Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-g9m7p"},"reconcileID":"a7e3947c-a7fc-40ba-a76e-c1644b176f14","JobStatus":"FAILED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:03:05.769Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-g9m7p"},"reconcileID":"a7e3947c-a7fc-40ba-a76e-c1644b176f14","name":"raycluster"}
{"level":"info","ts":"2026-11-03T15:03:05.769Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-g9m7p"},"reconcileID":"a7e3947c-a7fc-40ba-a76e-c1644b176f14","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"}}
{"level":"info","ts":"2026-11-03T15:03:05.772Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-g9m7p"},"reconcileID":"a7e3947c-a7fc-40ba-a76e-c1644b176f14","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:03:01Z","endTime":"2026-11-03T15:03:03Z"},"jobId":"fail-bbpjq","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-g9m7p.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-11-03 15:03:01,918\tINFO job_manager.py:583 -- Runtime env is setting up.\nRunning entrypoint for job fail-bbpjq: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-11-03T15:02:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:02:34Z","stateTransitionTimes":{"ready":"2026-11-03T15:02:34Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.31","serviceIP":"10.132.0.31","podName":"raycluster-head-b4s9s","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:02:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:02:34Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T14:58:42Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:03:01Z","endTime":"2026-11-03T15:03:03Z"},"jobId":"fail-bbpjq","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-g9m7p.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-11-03 15:03:01,918\tINFO job_manager.py:583 -- Runtime env is setting up.\nRunning entrypoint for job fail-bbpjq: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-11-03T15:02:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:02:34Z","stateTransitionTimes":{"ready":"2026-11-03T15:02:34Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.31","serviceIP":"10.132.0.31","podName":"raycluster-head-b4s9s","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:02:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:02:34Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T14:58:42Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:03:06.784Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"02845a55-fd3a-4295-8268-d9a45e7b3536","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:03:06.785Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"02845a55-fd3a-4295-8268-d9a45e7b3536","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-11-03T15:03:06.843Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"02845a55-fd3a-4295-8268-d9a45e7b3536"}
{"level":"info","ts":"2026-11-03T15:03:06.843Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"02845a55-fd3a-4295-8268-d9a45e7b3536","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-11-03T15:03:06.843Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"02845a55-fd3a-4295-8268-d9a45e7b3536","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:03:06.843Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"02845a55-fd3a-4295-8268-d9a45e7b3536","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:03:06.901Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"ee881581-b93f-40d5-9b2e-54ea16a148b4","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:03:06.901Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"ee881581-b93f-40d5-9b2e-54ea16a148b4","name":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:03:06.901Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"ee881581-b93f-40d5-9b2e-54ea16a148b4","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:03:06.901Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"ee881581-b93f-40d5-9b2e-54ea16a148b4","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:03:07.063Z","logger":"raycluster-resource","msg":"default","name":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:03:07.063Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"counter-5gq9h","namespace":"test-ns-mvz6h"}
{"level":"info","ts":"2026-11-03T15:03:07.234Z","logger":"raycluster-resource","msg":"validate create","name":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:03:07.288Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"ee881581-b93f-40d5-9b2e-54ea16a148b4","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:03:07.288Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"ee881581-b93f-40d5-9b2e-54ea16a148b4"}
{"level":"info","ts":"2026-11-03T15:03:07.288Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"ee881581-b93f-40d5-9b2e-54ea16a148b4","RayCluster":"counter-5gq9h","State":""}
{"level":"info","ts":"2026-11-03T15:03:07.288Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"ee881581-b93f-40d5-9b2e-54ea16a148b4","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:03:07.288Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-5gq9h","reconcileID":"06d40305-67ba-4e7c-95b0-6230f28c2ce1","namespacedName":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:03:07.288Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-5gq9h","reconcileID":"06d40305-67ba-4e7c-95b0-6230f28c2ce1","mode":"IntegratedOAuth","cluster":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:03:07.288Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-5gq9h","reconcileID":"06d40305-67ba-4e7c-95b0-6230f28c2ce1","cluster":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:03:07.288Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-5gq9h","reconcileID":"06d40305-67ba-4e7c-95b0-6230f28c2ce1","cluster":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:03:07.288Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"4ddbed40-3670-4d16-bcdc-ebe3f597e3a5"}
{"level":"info","ts":"2026-11-03T15:03:07.288Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"4ddbed40-3670-4d16-bcdc-ebe3f597e3a5"}
{"level":"info","ts":"2026-11-03T15:03:07.288Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"5443e380-4cee-4aae-b743-e0d471bf62a9","cluster":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:03:07.288Z","logger":"controllers.NetworkPolicy","msg":"Reconciling NetworkPolicies for RayCluster","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"9f16a3ac-c71b-42a6-b781-37ed83c1dc1a","cluster":"counter-5gq9h","namespace":"test-ns-mvz6h"}
{"level":"info","ts":"2026-11-03T15:03:07.288Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"a1cba7c6-9397-4359-aee3-78ae8a61956e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:03:07.288Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"a1cba7c6-9397-4359-aee3-78ae8a61956e","name":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:03:07.288Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"a1cba7c6-9397-4359-aee3-78ae8a61956e","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:03:07.289Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"a1cba7c6-9397-4359-aee3-78ae8a61956e"}
{"level":"info","ts":"2026-11-03T15:03:07.289Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"a1cba7c6-9397-4359-aee3-78ae8a61956e","RayCluster":"counter-5gq9h","State":""}
{"level":"info","ts":"2026-11-03T15:03:07.289Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"a1cba7c6-9397-4359-aee3-78ae8a61956e","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:03:07.340Z","logger":"raycluster-resource","msg":"default","name":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:03:07.340Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"counter-5gq9h","namespace":"test-ns-mvz6h"}
{"level":"info","ts":"2026-11-03T15:03:07.341Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"4ddbed40-3670-4d16-bcdc-ebe3f597e3a5","name":"counter-5gq9h-head-svc"}
{"level":"info","ts":"2026-11-03T15:03:07.341Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"4ddbed40-3670-4d16-bcdc-ebe3f597e3a5"}
{"level":"info","ts":"2026-11-03T15:03:07.387Z","logger":"controllers.NetworkPolicy","msg":"Successfully created NetworkPolicy","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"9f16a3ac-c71b-42a6-b781-37ed83c1dc1a","name":"counter-5gq9h-head"}
{"level":"info","ts":"2026-11-03T15:03:07.387Z","logger":"controllers.NetworkPolicy","msg":"Successfully reconciled NetworkPolicies for RayCluster","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"9f16a3ac-c71b-42a6-b781-37ed83c1dc1a","cluster":"counter-5gq9h","namespace":"test-ns-mvz6h"}
{"level":"info","ts":"2026-11-03T15:03:07.387Z","logger":"controllers.NetworkPolicy","msg":"Reconciling NetworkPolicies for RayCluster","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"3edc0092-146c-4d62-b55d-6414a321b628","cluster":"counter-5gq9h","namespace":"test-ns-mvz6h"}
{"level":"info","ts":"2026-11-03T15:03:07.390Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"4ddbed40-3670-4d16-bcdc-ebe3f597e3a5","error":"certificate ray-head-cert-counter-5gq9h not found: certificates.cert-manager.io \"ray-head-cert-counter-5gq9h\" not found"}
{"level":"error","ts":"2026-11-03T15:03:07.390Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"4ddbed40-3670-4d16-bcdc-ebe3f597e3a5","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-counter-5gq9h not found: certificates.cert-manager.io \"ray-head-cert-counter-5gq9h\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:03:07.390Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"4ddbed40-3670-4d16-bcdc-ebe3f597e3a5","error":"mTLS secrets not ready: certificate ray-head-cert-counter-5gq9h not found: certificates.cert-manager.io \"ray-head-cert-counter-5gq9h\" not found"}
{"level":"info","ts":"2026-11-03T15:03:07.391Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"4ddbed40-3670-4d16-bcdc-ebe3f597e3a5","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"counter-5gq9h","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:03:07.391Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"4ddbed40-3670-4d16-bcdc-ebe3f597e3a5","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"counter-5gq9h","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:03:07.391Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"4ddbed40-3670-4d16-bcdc-ebe3f597e3a5","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"counter-5gq9h","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:03:07.391Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"4ddbed40-3670-4d16-bcdc-ebe3f597e3a5","name":"counter-5gq9h","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""}],"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:03:07.401Z","logger":"raycluster-resource","msg":"validate update","name":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:03:07.441Z","logger":"controllers.NetworkPolicy","msg":"Successfully reconciled NetworkPolicies for RayCluster","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"3edc0092-146c-4d62-b55d-6414a321b628","cluster":"counter-5gq9h","namespace":"test-ns-mvz6h"}
{"level":"info","ts":"2026-11-03T15:03:07.450Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"4ddbed40-3670-4d16-bcdc-ebe3f597e3a5"}
{"level":"error","ts":"2026-11-03T15:03:07.450Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"4ddbed40-3670-4d16-bcdc-ebe3f597e3a5","error":"mTLS secrets not ready: certificate ray-head-cert-counter-5gq9h not found: certificates.cert-manager.io \"ray-head-cert-counter-5gq9h\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:03:07.450Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"512f9eab-b44e-4dcf-a94e-22bd199e501b"}
{"level":"info","ts":"2026-11-03T15:03:07.450Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"512f9eab-b44e-4dcf-a94e-22bd199e501b"}
{"level":"info","ts":"2026-11-03T15:03:07.450Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"512f9eab-b44e-4dcf-a94e-22bd199e501b","1 head service found":"counter-5gq9h-head-svc"}
{"level":"info","ts":"2026-11-03T15:03:07.450Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"512f9eab-b44e-4dcf-a94e-22bd199e501b"}
{"level":"info","ts":"2026-11-03T15:03:07.450Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"cd598807-2558-4466-80f4-308f4919cf79","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:03:07.450Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"cd598807-2558-4466-80f4-308f4919cf79","name":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:03:07.450Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"cd598807-2558-4466-80f4-308f4919cf79","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:03:07.450Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"cd598807-2558-4466-80f4-308f4919cf79"}
{"level":"info","ts":"2026-11-03T15:03:07.450Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"cd598807-2558-4466-80f4-308f4919cf79","RayCluster":"counter-5gq9h","State":""}
{"level":"info","ts":"2026-11-03T15:03:07.450Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"cd598807-2558-4466-80f4-308f4919cf79","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""}],"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:03:07.450Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"cd598807-2558-4466-80f4-308f4919cf79","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:03:07.457Z","logger":"raycluster-resource","msg":"default","name":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:03:07.458Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"counter-5gq9h","namespace":"test-ns-mvz6h"}
{"level":"info","ts":"2026-11-03T15:03:07.499Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"512f9eab-b44e-4dcf-a94e-22bd199e501b","error":"certificate ray-head-cert-counter-5gq9h not found: certificates.cert-manager.io \"ray-head-cert-counter-5gq9h\" not found"}
{"level":"error","ts":"2026-11-03T15:03:07.500Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"512f9eab-b44e-4dcf-a94e-22bd199e501b","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-counter-5gq9h not found: certificates.cert-manager.io \"ray-head-cert-counter-5gq9h\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:03:07.500Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"512f9eab-b44e-4dcf-a94e-22bd199e501b","error":"mTLS secrets not ready: certificate ray-head-cert-counter-5gq9h not found: certificates.cert-manager.io \"ray-head-cert-counter-5gq9h\" not found"}
{"level":"info","ts":"2026-11-03T15:03:07.500Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"512f9eab-b44e-4dcf-a94e-22bd199e501b","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"counter-5gq9h","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:03:07.500Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"512f9eab-b44e-4dcf-a94e-22bd199e501b","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"counter-5gq9h","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:03:07.500Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"512f9eab-b44e-4dcf-a94e-22bd199e501b","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"counter-5gq9h","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:03:07.500Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"512f9eab-b44e-4dcf-a94e-22bd199e501b","name":"counter-5gq9h","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""}],"observedGeneration":1}}
{"level":"error","ts":"2026-11-03T15:03:07.506Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-5gq9h","reconcileID":"06d40305-67ba-4e7c-95b0-6230f28c2ce1","error":"failed to add finalizer: Operation cannot be fulfilled on rayclusters.ray.io \"counter-5gq9h\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:167\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:03:07.506Z","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-5gq9h","reconcileID":"06d40305-67ba-4e7c-95b0-6230f28c2ce1"}
{"level":"error","ts":"2026-11-03T15:03:07.506Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-5gq9h","reconcileID":"06d40305-67ba-4e7c-95b0-6230f28c2ce1","error":"failed to add finalizer: Operation cannot be fulfilled on rayclusters.ray.io \"counter-5gq9h\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:03:07.506Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-5gq9h","reconcileID":"2d9dfeff-6bbd-4e9a-b905-4410c37bdabb","namespacedName":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:03:07.506Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-5gq9h","reconcileID":"2d9dfeff-6bbd-4e9a-b905-4410c37bdabb","mode":"IntegratedOAuth","cluster":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:03:07.506Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-5gq9h","reconcileID":"2d9dfeff-6bbd-4e9a-b905-4410c37bdabb","cluster":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:03:07.506Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-5gq9h","reconcileID":"2d9dfeff-6bbd-4e9a-b905-4410c37bdabb","cluster":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:03:07.509Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"d1b8a035-89c5-4ed0-a25e-2471096542f6","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:03:07.509Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"d1b8a035-89c5-4ed0-a25e-2471096542f6","name":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:03:07.509Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"d1b8a035-89c5-4ed0-a25e-2471096542f6","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:03:07.509Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"d1b8a035-89c5-4ed0-a25e-2471096542f6"}
{"level":"info","ts":"2026-11-03T15:03:07.509Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"d1b8a035-89c5-4ed0-a25e-2471096542f6","RayCluster":"counter-5gq9h","State":""}
{"level":"info","ts":"2026-11-03T15:03:07.509Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"d1b8a035-89c5-4ed0-a25e-2471096542f6","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""}],"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:03:07.554Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"512f9eab-b44e-4dcf-a94e-22bd199e501b","name":"counter-5gq9h","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-5gq9h\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-mvz6h","name":"counter-5gq9h"}}
{"level":"info","ts":"2026-11-03T15:03:07.554Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"512f9eab-b44e-4dcf-a94e-22bd199e501b"}
{"level":"error","ts":"2026-11-03T15:03:07.554Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"512f9eab-b44e-4dcf-a94e-22bd199e501b","error":"mTLS secrets not ready: certificate ray-head-cert-counter-5gq9h not found: certificates.cert-manager.io \"ray-head-cert-counter-5gq9h\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:03:07.559Z","logger":"raycluster-resource","msg":"default","name":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:03:07.559Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"counter-5gq9h","namespace":"test-ns-mvz6h"}
{"level":"info","ts":"2026-11-03T15:03:07.564Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"18703dbc-0ec1-4759-b5b2-68c55934ea78"}
{"level":"info","ts":"2026-11-03T15:03:07.564Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"18703dbc-0ec1-4759-b5b2-68c55934ea78"}
{"level":"info","ts":"2026-11-03T15:03:07.564Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"18703dbc-0ec1-4759-b5b2-68c55934ea78","1 head service found":"counter-5gq9h-head-svc"}
{"level":"info","ts":"2026-11-03T15:03:07.564Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"18703dbc-0ec1-4759-b5b2-68c55934ea78"}
{"level":"info","ts":"2026-11-03T15:03:07.611Z","logger":"raycluster-resource","msg":"validate update","name":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:03:07.613Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"18703dbc-0ec1-4759-b5b2-68c55934ea78","error":"certificate ray-head-cert-counter-5gq9h not found: certificates.cert-manager.io \"ray-head-cert-counter-5gq9h\" not found"}
{"level":"error","ts":"2026-11-03T15:03:07.613Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"18703dbc-0ec1-4759-b5b2-68c55934ea78","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-counter-5gq9h not found: certificates.cert-manager.io \"ray-head-cert-counter-5gq9h\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:03:07.613Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"18703dbc-0ec1-4759-b5b2-68c55934ea78","error":"mTLS secrets not ready: certificate ray-head-cert-counter-5gq9h not found: certificates.cert-manager.io \"ray-head-cert-counter-5gq9h\" not found"}
{"level":"info","ts":"2026-11-03T15:03:07.613Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"18703dbc-0ec1-4759-b5b2-68c55934ea78","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"counter-5gq9h","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:03:07.613Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"18703dbc-0ec1-4759-b5b2-68c55934ea78","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"counter-5gq9h","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:03:07.614Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"18703dbc-0ec1-4759-b5b2-68c55934ea78","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"counter-5gq9h","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:03:07.614Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"18703dbc-0ec1-4759-b5b2-68c55934ea78"}
{"level":"error","ts":"2026-11-03T15:03:07.614Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"18703dbc-0ec1-4759-b5b2-68c55934ea78","error":"mTLS secrets not ready: certificate ray-head-cert-counter-5gq9h not found: certificates.cert-manager.io \"ray-head-cert-counter-5gq9h\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:03:07.635Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"daff277f-d126-4a5b-9cb8-04099d6bef15"}
{"level":"info","ts":"2026-11-03T15:03:07.635Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"daff277f-d126-4a5b-9cb8-04099d6bef15"}
{"level":"info","ts":"2026-11-03T15:03:07.635Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"daff277f-d126-4a5b-9cb8-04099d6bef15","1 head service found":"counter-5gq9h-head-svc"}
{"level":"info","ts":"2026-11-03T15:03:07.635Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"daff277f-d126-4a5b-9cb8-04099d6bef15"}
{"level":"info","ts":"2026-11-03T15:03:07.666Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"3281f7dd-9f95-4cfe-94fe-520484abe1cb","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:03:07.666Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"3281f7dd-9f95-4cfe-94fe-520484abe1cb","name":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:03:07.666Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"3281f7dd-9f95-4cfe-94fe-520484abe1cb","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:03:07.666Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"3281f7dd-9f95-4cfe-94fe-520484abe1cb"}
{"level":"info","ts":"2026-11-03T15:03:07.666Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"3281f7dd-9f95-4cfe-94fe-520484abe1cb","RayCluster":"counter-5gq9h","State":""}
{"level":"info","ts":"2026-11-03T15:03:07.666Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"3281f7dd-9f95-4cfe-94fe-520484abe1cb","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""}],"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:03:07.685Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"daff277f-d126-4a5b-9cb8-04099d6bef15","error":"certificate ray-head-cert-counter-5gq9h not found: certificates.cert-manager.io \"ray-head-cert-counter-5gq9h\" not found"}
{"level":"error","ts":"2026-11-03T15:03:07.685Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"daff277f-d126-4a5b-9cb8-04099d6bef15","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-counter-5gq9h not found: certificates.cert-manager.io \"ray-head-cert-counter-5gq9h\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:03:07.685Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"daff277f-d126-4a5b-9cb8-04099d6bef15","error":"mTLS secrets not ready: certificate ray-head-cert-counter-5gq9h not found: certificates.cert-manager.io \"ray-head-cert-counter-5gq9h\" not found"}
{"level":"info","ts":"2026-11-03T15:03:07.685Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"daff277f-d126-4a5b-9cb8-04099d6bef15","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"counter-5gq9h","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:03:07.685Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"daff277f-d126-4a5b-9cb8-04099d6bef15","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"counter-5gq9h","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:03:07.685Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"daff277f-d126-4a5b-9cb8-04099d6bef15","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"counter-5gq9h","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:03:07.685Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"daff277f-d126-4a5b-9cb8-04099d6bef15"}
{"level":"error","ts":"2026-11-03T15:03:07.685Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"daff277f-d126-4a5b-9cb8-04099d6bef15","error":"mTLS secrets not ready: certificate ray-head-cert-counter-5gq9h not found: certificates.cert-manager.io \"ray-head-cert-counter-5gq9h\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:03:07.719Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-5gq9h","reconcileID":"2d9dfeff-6bbd-4e9a-b905-4410c37bdabb","name":"counter-5gq9h-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-11-03T15:03:07.725Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"5a37eb7a-e6f3-455b-8603-14bbafe95a6f"}
{"level":"info","ts":"2026-11-03T15:03:07.725Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"5a37eb7a-e6f3-455b-8603-14bbafe95a6f"}
{"level":"info","ts":"2026-11-03T15:03:07.725Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"5a37eb7a-e6f3-455b-8603-14bbafe95a6f","1 head service found":"counter-5gq9h-head-svc"}
{"level":"info","ts":"2026-11-03T15:03:07.725Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"5a37eb7a-e6f3-455b-8603-14bbafe95a6f"}
{"level":"info","ts":"2026-11-03T15:03:07.773Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-5gq9h","reconcileID":"2d9dfeff-6bbd-4e9a-b905-4410c37bdabb","name":"kuberay-gateway-access","namespace":"test-ns-mvz6h","operation":"created"}
{"level":"info","ts":"2026-11-03T15:03:07.775Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"5a37eb7a-e6f3-455b-8603-14bbafe95a6f","error":"certificate ray-head-cert-counter-5gq9h not found: certificates.cert-manager.io \"ray-head-cert-counter-5gq9h\" not found"}
{"level":"error","ts":"2026-11-03T15:03:07.775Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"5a37eb7a-e6f3-455b-8603-14bbafe95a6f","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-counter-5gq9h not found: certificates.cert-manager.io \"ray-head-cert-counter-5gq9h\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:03:07.775Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"5a37eb7a-e6f3-455b-8603-14bbafe95a6f","error":"mTLS secrets not ready: certificate ray-head-cert-counter-5gq9h not found: certificates.cert-manager.io \"ray-head-cert-counter-5gq9h\" not found"}
{"level":"info","ts":"2026-11-03T15:03:07.775Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"5a37eb7a-e6f3-455b-8603-14bbafe95a6f","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"counter-5gq9h","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:03:07.775Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"5a37eb7a-e6f3-455b-8603-14bbafe95a6f","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"counter-5gq9h","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:03:07.775Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"5a37eb7a-e6f3-455b-8603-14bbafe95a6f","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"counter-5gq9h","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:03:07.775Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"5a37eb7a-e6f3-455b-8603-14bbafe95a6f"}
{"level":"error","ts":"2026-11-03T15:03:07.775Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"5a37eb7a-e6f3-455b-8603-14bbafe95a6f","error":"mTLS secrets not ready: certificate ray-head-cert-counter-5gq9h not found: certificates.cert-manager.io \"ray-head-cert-counter-5gq9h\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:03:07.826Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-5gq9h","reconcileID":"2d9dfeff-6bbd-4e9a-b905-4410c37bdabb","name":"test-ns-mvz6h-counter-5gq9h","operation":"created"}
{"level":"info","ts":"2026-11-03T15:03:07.856Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"62782313-9c27-4067-9dec-fa2239ea16e8"}
{"level":"info","ts":"2026-11-03T15:03:07.856Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"62782313-9c27-4067-9dec-fa2239ea16e8"}
{"level":"info","ts":"2026-11-03T15:03:07.856Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"62782313-9c27-4067-9dec-fa2239ea16e8","1 head service found":"counter-5gq9h-head-svc"}
{"level":"info","ts":"2026-11-03T15:03:07.856Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"62782313-9c27-4067-9dec-fa2239ea16e8"}
{"level":"info","ts":"2026-11-03T15:03:07.878Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-5gq9h","reconcileID":"2d9dfeff-6bbd-4e9a-b905-4410c37bdabb","name":"kube-rbac-proxy-config-counter-5gq9h","operation":"created"}
{"level":"info","ts":"2026-11-03T15:03:07.905Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"62782313-9c27-4067-9dec-fa2239ea16e8","error":"certificate ray-head-cert-counter-5gq9h not found: certificates.cert-manager.io \"ray-head-cert-counter-5gq9h\" not found"}
{"level":"error","ts":"2026-11-03T15:03:07.905Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"62782313-9c27-4067-9dec-fa2239ea16e8","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-counter-5gq9h not found: certificates.cert-manager.io \"ray-head-cert-counter-5gq9h\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:03:07.905Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"62782313-9c27-4067-9dec-fa2239ea16e8","error":"mTLS secrets not ready: certificate ray-head-cert-counter-5gq9h not found: certificates.cert-manager.io \"ray-head-cert-counter-5gq9h\" not found"}
{"level":"info","ts":"2026-11-03T15:03:07.905Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"62782313-9c27-4067-9dec-fa2239ea16e8","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"counter-5gq9h","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:03:07.905Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"62782313-9c27-4067-9dec-fa2239ea16e8","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"counter-5gq9h","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:03:07.905Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"62782313-9c27-4067-9dec-fa2239ea16e8","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"counter-5gq9h","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:03:07.906Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"62782313-9c27-4067-9dec-fa2239ea16e8"}
{"level":"error","ts":"2026-11-03T15:03:07.906Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"62782313-9c27-4067-9dec-fa2239ea16e8","error":"mTLS secrets not ready: certificate ray-head-cert-counter-5gq9h not found: certificates.cert-manager.io \"ray-head-cert-counter-5gq9h\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-03T15:03:07.932Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-5gq9h","reconcileID":"2d9dfeff-6bbd-4e9a-b905-4410c37bdabb","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"counter-5gq9h\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:167\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:03:07.932Z","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-5gq9h","reconcileID":"2d9dfeff-6bbd-4e9a-b905-4410c37bdabb"}
{"level":"error","ts":"2026-11-03T15:03:07.932Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-5gq9h","reconcileID":"2d9dfeff-6bbd-4e9a-b905-4410c37bdabb","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"counter-5gq9h\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:03:07.933Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-5gq9h","reconcileID":"5753f43a-2bda-4bbf-8825-24885b31375b","namespacedName":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:03:07.933Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-5gq9h","reconcileID":"5753f43a-2bda-4bbf-8825-24885b31375b","mode":"IntegratedOAuth","cluster":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:03:07.933Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-5gq9h","reconcileID":"5753f43a-2bda-4bbf-8825-24885b31375b","cluster":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:03:07.992Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-5gq9h","reconcileID":"5753f43a-2bda-4bbf-8825-24885b31375b","cluster":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:03:07.993Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"ede5ce4f-062b-4a6f-85d6-f20ae101b2cd","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:03:07.993Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"ede5ce4f-062b-4a6f-85d6-f20ae101b2cd","name":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:03:07.993Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"ede5ce4f-062b-4a6f-85d6-f20ae101b2cd","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:03:07.993Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"ede5ce4f-062b-4a6f-85d6-f20ae101b2cd"}
{"level":"info","ts":"2026-11-03T15:03:07.993Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"ede5ce4f-062b-4a6f-85d6-f20ae101b2cd","RayCluster":"counter-5gq9h","State":""}
{"level":"info","ts":"2026-11-03T15:03:07.993Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"ede5ce4f-062b-4a6f-85d6-f20ae101b2cd","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:03:07.993Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"ede5ce4f-062b-4a6f-85d6-f20ae101b2cd","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:03:08.055Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"940104d9-6415-4248-9a7c-3d66b3473a24","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:03:08.055Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"940104d9-6415-4248-9a7c-3d66b3473a24","name":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:03:08.056Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"940104d9-6415-4248-9a7c-3d66b3473a24","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:03:08.056Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"940104d9-6415-4248-9a7c-3d66b3473a24"}
{"level":"info","ts":"2026-11-03T15:03:08.056Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"940104d9-6415-4248-9a7c-3d66b3473a24","RayCluster":"counter-5gq9h","State":""}
{"level":"info","ts":"2026-11-03T15:03:08.056Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"940104d9-6415-4248-9a7c-3d66b3473a24","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:03:08.066Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"5900e7fc-af4b-49ff-b6c9-02524d28872b"}
{"level":"info","ts":"2026-11-03T15:03:08.066Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"5900e7fc-af4b-49ff-b6c9-02524d28872b"}
{"level":"info","ts":"2026-11-03T15:03:08.066Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"5900e7fc-af4b-49ff-b6c9-02524d28872b","1 head service found":"counter-5gq9h-head-svc"}
{"level":"info","ts":"2026-11-03T15:03:08.066Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"5900e7fc-af4b-49ff-b6c9-02524d28872b"}
{"level":"info","ts":"2026-11-03T15:03:08.116Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"5900e7fc-af4b-49ff-b6c9-02524d28872b","error":"certificate ray-head-cert-counter-5gq9h is not ready at current generation 1"}
{"level":"error","ts":"2026-11-03T15:03:08.116Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"5900e7fc-af4b-49ff-b6c9-02524d28872b","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-counter-5gq9h is not ready at current generation 1","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:03:08.116Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"5900e7fc-af4b-49ff-b6c9-02524d28872b","error":"mTLS secrets not ready: certificate ray-head-cert-counter-5gq9h is not ready at current generation 1"}
{"level":"info","ts":"2026-11-03T15:03:08.116Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"5900e7fc-af4b-49ff-b6c9-02524d28872b","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"counter-5gq9h","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:03:08.116Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"5900e7fc-af4b-49ff-b6c9-02524d28872b","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"counter-5gq9h","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:03:08.116Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"5900e7fc-af4b-49ff-b6c9-02524d28872b","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"counter-5gq9h","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:03:08.116Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"5900e7fc-af4b-49ff-b6c9-02524d28872b"}
{"level":"error","ts":"2026-11-03T15:03:08.116Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"5900e7fc-af4b-49ff-b6c9-02524d28872b","error":"mTLS secrets not ready: certificate ray-head-cert-counter-5gq9h is not ready at current generation 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:03:08.265Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"5443e380-4cee-4aae-b743-e0d471bf62a9","certificate":"ray-head-cert-counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:03:08.265Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"5443e380-4cee-4aae-b743-e0d471bf62a9"}
{"level":"info","ts":"2026-11-03T15:03:08.265Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"4c8003cf-0db7-4983-8caa-cd08e89ec230","cluster":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:03:08.437Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"028f8731-5088-4759-9a62-7f3d5391849d"}
{"level":"info","ts":"2026-11-03T15:03:08.437Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"028f8731-5088-4759-9a62-7f3d5391849d"}
{"level":"info","ts":"2026-11-03T15:03:08.437Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"028f8731-5088-4759-9a62-7f3d5391849d","1 head service found":"counter-5gq9h-head-svc"}
{"level":"info","ts":"2026-11-03T15:03:08.437Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"028f8731-5088-4759-9a62-7f3d5391849d"}
{"level":"info","ts":"2026-11-03T15:03:08.487Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"028f8731-5088-4759-9a62-7f3d5391849d","error":"certificate ray-head-cert-counter-5gq9h is not ready at current generation 1"}
{"level":"error","ts":"2026-11-03T15:03:08.487Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"028f8731-5088-4759-9a62-7f3d5391849d","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-counter-5gq9h is not ready at current generation 1","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:03:08.487Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"028f8731-5088-4759-9a62-7f3d5391849d","error":"mTLS secrets not ready: certificate ray-head-cert-counter-5gq9h is not ready at current generation 1"}
{"level":"info","ts":"2026-11-03T15:03:08.487Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"028f8731-5088-4759-9a62-7f3d5391849d","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"counter-5gq9h","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:03:08.487Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"028f8731-5088-4759-9a62-7f3d5391849d","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"counter-5gq9h","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:03:08.487Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"028f8731-5088-4759-9a62-7f3d5391849d","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"counter-5gq9h","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:03:08.487Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"028f8731-5088-4759-9a62-7f3d5391849d"}
{"level":"error","ts":"2026-11-03T15:03:08.487Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"028f8731-5088-4759-9a62-7f3d5391849d","error":"mTLS secrets not ready: certificate ray-head-cert-counter-5gq9h is not ready at current generation 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:03:08.713Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"4c8003cf-0db7-4983-8caa-cd08e89ec230","certificate":"ray-head-cert-counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:03:08.713Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"4c8003cf-0db7-4983-8caa-cd08e89ec230"}
{"level":"info","ts":"2026-11-03T15:03:08.773Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-g9m7p"},"reconcileID":"4f86dc96-ce0f-4cfa-8c9a-76ebfab6fda5","JobStatus":"FAILED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:03:08.773Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-g9m7p"},"reconcileID":"4f86dc96-ce0f-4cfa-8c9a-76ebfab6fda5","name":"raycluster"}
{"level":"info","ts":"2026-11-03T15:03:08.773Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-g9m7p"},"reconcileID":"4f86dc96-ce0f-4cfa-8c9a-76ebfab6fda5","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"}}
{"level":"info","ts":"2026-11-03T15:03:08.777Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-g9m7p"},"reconcileID":"4f86dc96-ce0f-4cfa-8c9a-76ebfab6fda5","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:03:01Z","endTime":"2026-11-03T15:03:03Z"},"jobId":"fail-bbpjq","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-g9m7p.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-11-03 15:03:01,918\tINFO job_manager.py:583 -- Runtime env is setting up.\nRunning entrypoint for job fail-bbpjq: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-11-03T15:02:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:02:34Z","stateTransitionTimes":{"ready":"2026-11-03T15:02:34Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.31","serviceIP":"10.132.0.31","podName":"raycluster-head-b4s9s","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:02:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:02:34Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T14:58:42Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:03:01Z","endTime":"2026-11-03T15:03:03Z"},"jobId":"fail-bbpjq","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-g9m7p.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-11-03 15:03:01,918\tINFO job_manager.py:583 -- Runtime env is setting up.\nRunning entrypoint for job fail-bbpjq: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-11-03T15:02:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:02:34Z","stateTransitionTimes":{"ready":"2026-11-03T15:02:34Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.31","serviceIP":"10.132.0.31","podName":"raycluster-head-b4s9s","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:02:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:02:34Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T14:58:42Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:03:09.128Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"2737aca9-75b1-48c4-8850-3483635548dc"}
{"level":"info","ts":"2026-11-03T15:03:09.128Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"2737aca9-75b1-48c4-8850-3483635548dc"}
{"level":"info","ts":"2026-11-03T15:03:09.128Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"2737aca9-75b1-48c4-8850-3483635548dc","1 head service found":"counter-5gq9h-head-svc"}
{"level":"info","ts":"2026-11-03T15:03:09.128Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"2737aca9-75b1-48c4-8850-3483635548dc"}
{"level":"info","ts":"2026-11-03T15:03:09.178Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"2737aca9-75b1-48c4-8850-3483635548dc","error":"certificate ray-head-cert-counter-5gq9h is not ready at current generation 1"}
{"level":"error","ts":"2026-11-03T15:03:09.178Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"2737aca9-75b1-48c4-8850-3483635548dc","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-counter-5gq9h is not ready at current generation 1","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:03:09.178Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"2737aca9-75b1-48c4-8850-3483635548dc","error":"mTLS secrets not ready: certificate ray-head-cert-counter-5gq9h is not ready at current generation 1"}
{"level":"info","ts":"2026-11-03T15:03:09.178Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"2737aca9-75b1-48c4-8850-3483635548dc","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"counter-5gq9h","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:03:09.178Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"2737aca9-75b1-48c4-8850-3483635548dc","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"counter-5gq9h","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:03:09.178Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"2737aca9-75b1-48c4-8850-3483635548dc","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"counter-5gq9h","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:03:09.179Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"2737aca9-75b1-48c4-8850-3483635548dc"}
{"level":"error","ts":"2026-11-03T15:03:09.179Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"2737aca9-75b1-48c4-8850-3483635548dc","error":"mTLS secrets not ready: certificate ray-head-cert-counter-5gq9h is not ready at current generation 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:03:10.459Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"83635f90-541b-4071-ae1b-150c439032f7"}
{"level":"info","ts":"2026-11-03T15:03:10.459Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"83635f90-541b-4071-ae1b-150c439032f7"}
{"level":"info","ts":"2026-11-03T15:03:10.459Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"83635f90-541b-4071-ae1b-150c439032f7","1 head service found":"counter-5gq9h-head-svc"}
{"level":"info","ts":"2026-11-03T15:03:10.460Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"83635f90-541b-4071-ae1b-150c439032f7"}
{"level":"info","ts":"2026-11-03T15:03:10.560Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"83635f90-541b-4071-ae1b-150c439032f7","mode":"IntegratedOAuth","cluster":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:03:10.560Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"83635f90-541b-4071-ae1b-150c439032f7","cluster":"counter-5gq9h","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-11-03T15:03:10.560Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"83635f90-541b-4071-ae1b-150c439032f7","cluster":"counter-5gq9h","authType":"IntegratedOAuth","serviceAccount":"counter-5gq9h-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-11-03T15:03:10.560Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"83635f90-541b-4071-ae1b-150c439032f7","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"counter-5gq9h","ray.io/group":"headgroup","ray.io/identifier":"counter-5gq9h-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-11-03T15:03:10.560Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"83635f90-541b-4071-ae1b-150c439032f7","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","num-cpus":"8","num-gpus":"4","resources":"'{\"R1\": 4}'"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-11-03T15:03:10.560Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"83635f90-541b-4071-ae1b-150c439032f7","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=8  --num-gpus=4  --resources='{\"R1\": 4}' "}
{"level":"info","ts":"2026-11-03T15:03:10.560Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"83635f90-541b-4071-ae1b-150c439032f7","rayNodeType":"head","generatedCmd":"ulimit -n ${RAY_START_ULIMIT_OPEN_FILES:-65536}; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=8  --num-gpus=4  --resources='{\"R1\": 4}' "}
{"level":"info","ts":"2026-11-03T15:03:10.560Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"83635f90-541b-4071-ae1b-150c439032f7","enabled":true}
{"level":"info","ts":"2026-11-03T15:03:10.638Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"83635f90-541b-4071-ae1b-150c439032f7","name":"counter-5gq9h-head-9twx4"}
{"level":"info","ts":"2026-11-03T15:03:10.638Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"83635f90-541b-4071-ae1b-150c439032f7","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"counter-5gq9h","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:03:10.638Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"83635f90-541b-4071-ae1b-150c439032f7","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"counter-5gq9h","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:03:10.638Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"83635f90-541b-4071-ae1b-150c439032f7","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"counter-5gq9h","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:03:10.638Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"83635f90-541b-4071-ae1b-150c439032f7","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:03:10.638Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"83635f90-541b-4071-ae1b-150c439032f7","seconds":300}
{"level":"info","ts":"2026-11-03T15:03:10.671Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"7fa2ba76-303c-4512-b536-ed9e25c245f0"}
{"level":"info","ts":"2026-11-03T15:03:10.671Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"7fa2ba76-303c-4512-b536-ed9e25c245f0"}
{"level":"info","ts":"2026-11-03T15:03:10.671Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"7fa2ba76-303c-4512-b536-ed9e25c245f0","1 head service found":"counter-5gq9h-head-svc"}
{"level":"info","ts":"2026-11-03T15:03:10.671Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"7fa2ba76-303c-4512-b536-ed9e25c245f0","Found 1 head Pod":"counter-5gq9h-head-9twx4","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:03:10.671Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"7fa2ba76-303c-4512-b536-ed9e25c245f0","head Pod":"counter-5gq9h-head-9twx4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-5gq9h-head-9twx4. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:03:10.671Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"7fa2ba76-303c-4512-b536-ed9e25c245f0","podName":"counter-5gq9h-head-9twx4","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:03:10.671Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"7fa2ba76-303c-4512-b536-ed9e25c245f0","podName":"counter-5gq9h-head-9twx4","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:03:10.671Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"7fa2ba76-303c-4512-b536-ed9e25c245f0","name":"counter-5gq9h","old status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1},"new status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:03:10.732Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"821510e7-84fb-4554-8ed6-b6e9f1caa0ee"}
{"level":"info","ts":"2026-11-03T15:03:10.732Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"821510e7-84fb-4554-8ed6-b6e9f1caa0ee"}
{"level":"info","ts":"2026-11-03T15:03:10.732Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"821510e7-84fb-4554-8ed6-b6e9f1caa0ee","1 head service found":"counter-5gq9h-head-svc"}
{"level":"info","ts":"2026-11-03T15:03:10.732Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"821510e7-84fb-4554-8ed6-b6e9f1caa0ee","Found 1 head Pod":"counter-5gq9h-head-9twx4","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:03:10.732Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"821510e7-84fb-4554-8ed6-b6e9f1caa0ee","head Pod":"counter-5gq9h-head-9twx4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-5gq9h-head-9twx4. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:03:10.732Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"821510e7-84fb-4554-8ed6-b6e9f1caa0ee","podName":"counter-5gq9h-head-9twx4","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:03:10.732Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"821510e7-84fb-4554-8ed6-b6e9f1caa0ee","podName":"counter-5gq9h-head-9twx4","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:03:10.732Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"a31caa7f-d271-4a46-b0d0-0f7cddf0dddc","cluster":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:03:10.732Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"821510e7-84fb-4554-8ed6-b6e9f1caa0ee","name":"counter-5gq9h","old status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1},"new status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:03:10.732Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"a32e66c7-e43b-4101-9db0-9b88d2cb07fd","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:03:10.732Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"a32e66c7-e43b-4101-9db0-9b88d2cb07fd","name":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:03:10.732Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"a32e66c7-e43b-4101-9db0-9b88d2cb07fd","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:03:10.732Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"a32e66c7-e43b-4101-9db0-9b88d2cb07fd"}
{"level":"info","ts":"2026-11-03T15:03:10.732Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"a32e66c7-e43b-4101-9db0-9b88d2cb07fd","RayCluster":"counter-5gq9h","State":""}
{"level":"info","ts":"2026-11-03T15:03:10.732Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"a32e66c7-e43b-4101-9db0-9b88d2cb07fd","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:07Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:03:10.732Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"a32e66c7-e43b-4101-9db0-9b88d2cb07fd","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:03:10.787Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"821510e7-84fb-4554-8ed6-b6e9f1caa0ee","name":"counter-5gq9h","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-5gq9h\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-mvz6h","name":"counter-5gq9h"}}
{"level":"info","ts":"2026-11-03T15:03:10.787Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"821510e7-84fb-4554-8ed6-b6e9f1caa0ee"}
{"level":"error","ts":"2026-11-03T15:03:10.787Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"821510e7-84fb-4554-8ed6-b6e9f1caa0ee","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-5gq9h\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:03:10.787Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"0f4740c8-6f58-487a-9490-9ad7745b2452"}
{"level":"info","ts":"2026-11-03T15:03:10.787Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"0f4740c8-6f58-487a-9490-9ad7745b2452"}
{"level":"info","ts":"2026-11-03T15:03:10.787Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"0f4740c8-6f58-487a-9490-9ad7745b2452","1 head service found":"counter-5gq9h-head-svc"}
{"level":"info","ts":"2026-11-03T15:03:10.787Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"0f4740c8-6f58-487a-9490-9ad7745b2452","Found 1 head Pod":"counter-5gq9h-head-9twx4","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:03:10.787Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"0f4740c8-6f58-487a-9490-9ad7745b2452","head Pod":"counter-5gq9h-head-9twx4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-5gq9h-head-9twx4. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:03:10.787Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"0f4740c8-6f58-487a-9490-9ad7745b2452","podName":"counter-5gq9h-head-9twx4","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:03:10.787Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"0f4740c8-6f58-487a-9490-9ad7745b2452","podName":"counter-5gq9h-head-9twx4","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:03:10.787Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"0f4740c8-6f58-487a-9490-9ad7745b2452","name":"counter-5gq9h","old status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1},"new status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:03:10.792Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"4f83b8e6-31f7-4464-a02a-98bf03fa63c8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:03:10.792Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"4f83b8e6-31f7-4464-a02a-98bf03fa63c8","name":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:03:10.792Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"4f83b8e6-31f7-4464-a02a-98bf03fa63c8","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:03:10.792Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"4f83b8e6-31f7-4464-a02a-98bf03fa63c8"}
{"level":"info","ts":"2026-11-03T15:03:10.792Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"4f83b8e6-31f7-4464-a02a-98bf03fa63c8","RayCluster":"counter-5gq9h","State":""}
{"level":"info","ts":"2026-11-03T15:03:10.792Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"4f83b8e6-31f7-4464-a02a-98bf03fa63c8","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:03:10.848Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"260a9a5f-5e73-4139-978d-4f19d9f43615","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:03:10.848Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"260a9a5f-5e73-4139-978d-4f19d9f43615","name":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:03:10.848Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"260a9a5f-5e73-4139-978d-4f19d9f43615","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:03:10.848Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"260a9a5f-5e73-4139-978d-4f19d9f43615"}
{"level":"info","ts":"2026-11-03T15:03:10.848Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"260a9a5f-5e73-4139-978d-4f19d9f43615","RayCluster":"counter-5gq9h","State":""}
{"level":"info","ts":"2026-11-03T15:03:10.848Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"260a9a5f-5e73-4139-978d-4f19d9f43615","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:03:10.848Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"260a9a5f-5e73-4139-978d-4f19d9f43615","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:03:10.910Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"63d4fa85-4e3f-4fe7-a87a-e5aa35713a90","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:03:10.910Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"63d4fa85-4e3f-4fe7-a87a-e5aa35713a90","name":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:03:10.910Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"63d4fa85-4e3f-4fe7-a87a-e5aa35713a90","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:03:10.911Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"63d4fa85-4e3f-4fe7-a87a-e5aa35713a90"}
{"level":"info","ts":"2026-11-03T15:03:10.911Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"63d4fa85-4e3f-4fe7-a87a-e5aa35713a90","RayCluster":"counter-5gq9h","State":""}
{"level":"info","ts":"2026-11-03T15:03:10.911Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"63d4fa85-4e3f-4fe7-a87a-e5aa35713a90","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:03:11.371Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"e0a67dff-bb9f-42fa-a6ff-610013768c45"}
{"level":"info","ts":"2026-11-03T15:03:11.371Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"e0a67dff-bb9f-42fa-a6ff-610013768c45"}
{"level":"info","ts":"2026-11-03T15:03:11.371Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"e0a67dff-bb9f-42fa-a6ff-610013768c45","1 head service found":"counter-5gq9h-head-svc"}
{"level":"info","ts":"2026-11-03T15:03:11.371Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"e0a67dff-bb9f-42fa-a6ff-610013768c45","Found 1 head Pod":"counter-5gq9h-head-9twx4","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:03:11.371Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"e0a67dff-bb9f-42fa-a6ff-610013768c45","head Pod":"counter-5gq9h-head-9twx4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-5gq9h-head-9twx4. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:03:11.371Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"e0a67dff-bb9f-42fa-a6ff-610013768c45","podName":"counter-5gq9h-head-9twx4","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:03:11.372Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"e0a67dff-bb9f-42fa-a6ff-610013768c45","podName":"counter-5gq9h-head-9twx4","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:03:11.372Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"e0a67dff-bb9f-42fa-a6ff-610013768c45","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:03:11.372Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"e0a67dff-bb9f-42fa-a6ff-610013768c45","seconds":300}
{"level":"info","ts":"2026-11-03T15:03:11.549Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"56c82db1-883f-4158-937f-51c5f53f4783","cluster":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:03:11.717Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"5b0c21ce-3e4f-41bd-bf94-f468b1731417"}
{"level":"info","ts":"2026-11-03T15:03:11.717Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"5b0c21ce-3e4f-41bd-bf94-f468b1731417"}
{"level":"info","ts":"2026-11-03T15:03:11.717Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"5b0c21ce-3e4f-41bd-bf94-f468b1731417","1 head service found":"counter-5gq9h-head-svc"}
{"level":"info","ts":"2026-11-03T15:03:11.717Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"5b0c21ce-3e4f-41bd-bf94-f468b1731417","Found 1 head Pod":"counter-5gq9h-head-9twx4","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:03:11.717Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"5b0c21ce-3e4f-41bd-bf94-f468b1731417","head Pod":"counter-5gq9h-head-9twx4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-5gq9h-head-9twx4. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:03:11.717Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"5b0c21ce-3e4f-41bd-bf94-f468b1731417","podName":"counter-5gq9h-head-9twx4","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:03:11.717Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"5b0c21ce-3e4f-41bd-bf94-f468b1731417","podName":"counter-5gq9h-head-9twx4","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:03:11.717Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"5b0c21ce-3e4f-41bd-bf94-f468b1731417","name":"counter-5gq9h","old status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1},"new status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:03:11.777Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"6e166973-bd00-4d65-b34a-077a8f330283"}
{"level":"info","ts":"2026-11-03T15:03:11.777Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"6e166973-bd00-4d65-b34a-077a8f330283"}
{"level":"info","ts":"2026-11-03T15:03:11.777Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"6e166973-bd00-4d65-b34a-077a8f330283","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:03:11.777Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"6e166973-bd00-4d65-b34a-077a8f330283","Found 1 head Pod":"raycluster-head-b4s9s","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:03:11.777Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"6e166973-bd00-4d65-b34a-077a8f330283","head Pod":"raycluster-head-b4s9s","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-b4s9s. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:03:11.777Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"6e166973-bd00-4d65-b34a-077a8f330283","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:03:11.777Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"6e166973-bd00-4d65-b34a-077a8f330283","worker Pod":"raycluster-small-group-worker-qrhm7","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-qrhm7. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:03:11.777Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"6e166973-bd00-4d65-b34a-077a8f330283","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:03:11.777Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"6e166973-bd00-4d65-b34a-077a8f330283","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:03:11.777Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"6e166973-bd00-4d65-b34a-077a8f330283","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:03:11.777Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"6e166973-bd00-4d65-b34a-077a8f330283","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:03:11.777Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"6e166973-bd00-4d65-b34a-077a8f330283","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:03:11.777Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"6e166973-bd00-4d65-b34a-077a8f330283","seconds":300}
{"level":"info","ts":"2026-11-03T15:03:11.777Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"3d282494-a47d-4e2c-8546-250c8c7dc00c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:03:11.777Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"3d282494-a47d-4e2c-8546-250c8c7dc00c","name":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:03:11.777Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"3d282494-a47d-4e2c-8546-250c8c7dc00c","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:03:11.778Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"3d282494-a47d-4e2c-8546-250c8c7dc00c"}
{"level":"info","ts":"2026-11-03T15:03:11.778Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"3d282494-a47d-4e2c-8546-250c8c7dc00c","RayCluster":"counter-5gq9h","State":""}
{"level":"info","ts":"2026-11-03T15:03:11.778Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"3d282494-a47d-4e2c-8546-250c8c7dc00c","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:03:11.778Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"3d282494-a47d-4e2c-8546-250c8c7dc00c","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:03:11.838Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-g9m7p"},"reconcileID":"b7d8c3a5-c82d-45fe-b318-2728397dc81e","JobStatus":"FAILED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:03:11.839Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-g9m7p"},"reconcileID":"b7d8c3a5-c82d-45fe-b318-2728397dc81e","name":"raycluster"}
{"level":"info","ts":"2026-11-03T15:03:11.839Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-g9m7p"},"reconcileID":"b7d8c3a5-c82d-45fe-b318-2728397dc81e","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"}}
{"level":"info","ts":"2026-11-03T15:03:11.842Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-g9m7p"},"reconcileID":"b7d8c3a5-c82d-45fe-b318-2728397dc81e","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:03:01Z","endTime":"2026-11-03T15:03:03Z"},"jobId":"fail-bbpjq","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-g9m7p.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-11-03 15:03:01,918\tINFO job_manager.py:583 -- Runtime env is setting up.\nRunning entrypoint for job fail-bbpjq: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-11-03T15:02:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:02:34Z","stateTransitionTimes":{"ready":"2026-11-03T15:02:34Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.31","serviceIP":"10.132.0.31","podName":"raycluster-head-b4s9s","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:02:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:02:34Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T14:58:42Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:03:01Z","endTime":"2026-11-03T15:03:03Z"},"jobId":"fail-bbpjq","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-g9m7p.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-11-03 15:03:01,918\tINFO job_manager.py:583 -- Runtime env is setting up.\nRunning entrypoint for job fail-bbpjq: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-11-03T15:02:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:02:34Z","stateTransitionTimes":{"ready":"2026-11-03T15:02:34Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.31","serviceIP":"10.132.0.31","podName":"raycluster-head-b4s9s","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:02:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:02:34Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T14:58:42Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:03:11.842Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"bf014110-cf78-4eee-95fe-c2250649fdc6","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:03:11.842Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"bf014110-cf78-4eee-95fe-c2250649fdc6","name":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:03:11.842Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"bf014110-cf78-4eee-95fe-c2250649fdc6","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:03:11.842Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"bf014110-cf78-4eee-95fe-c2250649fdc6"}
{"level":"info","ts":"2026-11-03T15:03:11.842Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"bf014110-cf78-4eee-95fe-c2250649fdc6","RayCluster":"counter-5gq9h","State":""}
{"level":"info","ts":"2026-11-03T15:03:11.842Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"bf014110-cf78-4eee-95fe-c2250649fdc6","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:03:11.853Z","logger":"controllers.RayClusterMTLS","msg":"Updating head certificate","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"56c82db1-883f-4158-937f-51c5f53f4783","certificate":"ray-head-cert-counter-5gq9h","ipAddresses":["10.134.0.27","127.0.0.1"]}
{"level":"info","ts":"2026-11-03T15:03:11.879Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ee66043e-8a52-4bdf-aef7-6269f8b2cb9a"}
{"level":"info","ts":"2026-11-03T15:03:11.879Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ee66043e-8a52-4bdf-aef7-6269f8b2cb9a"}
{"level":"info","ts":"2026-11-03T15:03:11.879Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ee66043e-8a52-4bdf-aef7-6269f8b2cb9a","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:03:11.879Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ee66043e-8a52-4bdf-aef7-6269f8b2cb9a","Found 1 head Pod":"raycluster-head-b4s9s","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:03:11.879Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ee66043e-8a52-4bdf-aef7-6269f8b2cb9a","head Pod":"raycluster-head-b4s9s","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-b4s9s. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:03:11.879Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ee66043e-8a52-4bdf-aef7-6269f8b2cb9a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:03:11.879Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ee66043e-8a52-4bdf-aef7-6269f8b2cb9a","worker Pod":"raycluster-small-group-worker-qrhm7","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-qrhm7. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:03:11.879Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ee66043e-8a52-4bdf-aef7-6269f8b2cb9a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:03:11.879Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ee66043e-8a52-4bdf-aef7-6269f8b2cb9a","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:03:11.879Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ee66043e-8a52-4bdf-aef7-6269f8b2cb9a","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:03:11.879Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ee66043e-8a52-4bdf-aef7-6269f8b2cb9a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:03:11.879Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ee66043e-8a52-4bdf-aef7-6269f8b2cb9a","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:03:11.879Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ee66043e-8a52-4bdf-aef7-6269f8b2cb9a","seconds":300}
{"level":"info","ts":"2026-11-03T15:03:11.908Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"18d31a23-cf02-4202-97bb-931351707604"}
{"level":"info","ts":"2026-11-03T15:03:11.908Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"18d31a23-cf02-4202-97bb-931351707604"}
{"level":"info","ts":"2026-11-03T15:03:11.908Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"18d31a23-cf02-4202-97bb-931351707604","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:03:11.908Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"18d31a23-cf02-4202-97bb-931351707604","Found 1 head Pod":"raycluster-head-b4s9s","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:03:11.908Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"18d31a23-cf02-4202-97bb-931351707604","head Pod":"raycluster-head-b4s9s","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-b4s9s. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:03:11.908Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"18d31a23-cf02-4202-97bb-931351707604","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:03:11.908Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"18d31a23-cf02-4202-97bb-931351707604","worker Pod":"raycluster-small-group-worker-qrhm7","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-small-group-worker-qrhm7. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:03:11.908Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"18d31a23-cf02-4202-97bb-931351707604","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:03:11.908Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"18d31a23-cf02-4202-97bb-931351707604","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:03:11.908Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"18d31a23-cf02-4202-97bb-931351707604","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:03:11.908Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"18d31a23-cf02-4202-97bb-931351707604","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:03:11.908Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"18d31a23-cf02-4202-97bb-931351707604","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:03:11.908Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"18d31a23-cf02-4202-97bb-931351707604","seconds":300}
{"level":"info","ts":"2026-11-03T15:03:12.266Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"56c82db1-883f-4158-937f-51c5f53f4783","certificate":"ray-head-cert-counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:03:12.266Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"56c82db1-883f-4158-937f-51c5f53f4783"}
{"level":"info","ts":"2026-11-03T15:03:12.266Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"52f9604b-3b47-4e09-b12c-9117ca6b8911","cluster":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:03:12.612Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-g9m7p"},"reconcileID":"73584958-00bf-4487-9ec2-1b67fc4b9cc9","JobStatus":"FAILED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:03:12.612Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-g9m7p"},"reconcileID":"73584958-00bf-4487-9ec2-1b67fc4b9cc9","name":"raycluster"}
{"level":"info","ts":"2026-11-03T15:03:12.612Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-g9m7p"},"reconcileID":"73584958-00bf-4487-9ec2-1b67fc4b9cc9","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"}}
{"level":"error","ts":"2026-11-03T15:03:12.617Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-g9m7p"},"reconcileID":"73584958-00bf-4487-9ec2-1b67fc4b9cc9","JobId":"fail-bbpjq","error":"Get \"http://raycluster-head-svc.test-ns-g9m7p.svc.cluster.local:8265/api/jobs/fail-bbpjq\": dial tcp 10.132.0.31:8265: connect: connection refused","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:321\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-03T15:03:12.617Z","logger":"controllers.RayJob","msg":"Job status check began failing.","RayJob":{"name":"fail","namespace":"test-ns-g9m7p"},"reconcileID":"73584958-00bf-4487-9ec2-1b67fc4b9cc9","JobId":"fail-bbpjq","error":"Get \"http://raycluster-head-svc.test-ns-g9m7p.svc.cluster.local:8265/api/jobs/fail-bbpjq\": dial tcp 10.132.0.31:8265: connect: connection refused","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.recordJobStatusCheckFailure\n\t/workspace/controllers/ray/rayjob_controller.go:1381\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:322\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:03:12.617Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-g9m7p"},"reconcileID":"73584958-00bf-4487-9ec2-1b67fc4b9cc9","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:03:01Z","endTime":"2026-11-03T15:03:03Z"},"jobId":"fail-bbpjq","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-g9m7p.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-11-03 15:03:01,918\tINFO job_manager.py:583 -- Runtime env is setting up.\nRunning entrypoint for job fail-bbpjq: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-11-03T15:02:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:02:34Z","stateTransitionTimes":{"ready":"2026-11-03T15:02:34Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.31","serviceIP":"10.132.0.31","podName":"raycluster-head-b4s9s","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:02:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:02:34Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T14:58:42Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:03:01Z","endTime":"2026-11-03T15:03:03Z"},"jobId":"fail-bbpjq","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-g9m7p.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-11-03 15:03:01,918\tINFO job_manager.py:583 -- Runtime env is setting up.\nRunning entrypoint for job fail-bbpjq: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-11-03T15:02:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:02:34Z","stateTransitionTimes":{"ready":"2026-11-03T15:02:34Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.31","serviceIP":"10.132.0.31","podName":"raycluster-head-b4s9s","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:02:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:02:34Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T14:58:42Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"jobStatusCheckFailureStartTime":"2026-11-03T15:03:12Z"}}
{"level":"info","ts":"2026-11-03T15:03:12.617Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-g9m7p"},"reconcileID":"73584958-00bf-4487-9ec2-1b67fc4b9cc9","old JobStatus":"FAILED","new JobStatus":"FAILED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-11-03T15:03:12.677Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-g9m7p"},"reconcileID":"ace119fe-2c7a-4e17-beaa-2762ee18a926","JobStatus":"FAILED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:03:12.677Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-g9m7p"},"reconcileID":"ace119fe-2c7a-4e17-beaa-2762ee18a926","name":"raycluster"}
{"level":"info","ts":"2026-11-03T15:03:12.677Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-g9m7p"},"reconcileID":"ace119fe-2c7a-4e17-beaa-2762ee18a926","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"}}
{"level":"error","ts":"2026-11-03T15:03:12.679Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-g9m7p"},"reconcileID":"ace119fe-2c7a-4e17-beaa-2762ee18a926","JobId":"fail-bbpjq","error":"Get \"http://raycluster-head-svc.test-ns-g9m7p.svc.cluster.local:8265/api/jobs/fail-bbpjq\": dial tcp 10.132.0.31:8265: connect: connection refused","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:321\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-03T15:03:12.679Z","logger":"controllers.RayJob","msg":"Job status check began failing.","RayJob":{"name":"fail","namespace":"test-ns-g9m7p"},"reconcileID":"ace119fe-2c7a-4e17-beaa-2762ee18a926","JobId":"fail-bbpjq","error":"Get \"http://raycluster-head-svc.test-ns-g9m7p.svc.cluster.local:8265/api/jobs/fail-bbpjq\": dial tcp 10.132.0.31:8265: connect: connection refused","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.recordJobStatusCheckFailure\n\t/workspace/controllers/ray/rayjob_controller.go:1381\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:322\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:03:12.679Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-g9m7p"},"reconcileID":"ace119fe-2c7a-4e17-beaa-2762ee18a926","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:03:01Z","endTime":"2026-11-03T15:03:03Z"},"jobId":"fail-bbpjq","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-g9m7p.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-11-03 15:03:01,918\tINFO job_manager.py:583 -- Runtime env is setting up.\nRunning entrypoint for job fail-bbpjq: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-11-03T15:02:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:02:34Z","stateTransitionTimes":{"ready":"2026-11-03T15:02:34Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.31","serviceIP":"10.132.0.31","podName":"raycluster-head-b4s9s","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:02:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:02:34Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T14:58:42Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:03:01Z","endTime":"2026-11-03T15:03:03Z"},"jobId":"fail-bbpjq","rayClusterName":"raycluster","dashboardURL":"raycluster-head-svc.test-ns-g9m7p.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-11-03 15:03:01,918\tINFO job_manager.py:583 -- Runtime env is setting up.\nRunning entrypoint for job fail-bbpjq: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-11-03T15:02:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:02:34Z","stateTransitionTimes":{"ready":"2026-11-03T15:02:34Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.31","serviceIP":"10.132.0.31","podName":"raycluster-head-b4s9s","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:02:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:02:34Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T14:58:42Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"jobStatusCheckFailureStartTime":"2026-11-03T15:03:12Z"}}
{"level":"info","ts":"2026-11-03T15:03:12.679Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-g9m7p"},"reconcileID":"ace119fe-2c7a-4e17-beaa-2762ee18a926","old JobStatus":"FAILED","new JobStatus":"FAILED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-11-03T15:03:12.733Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"fail","namespace":"test-ns-g9m7p"},"reconcileID":"ace119fe-2c7a-4e17-beaa-2762ee18a926","error":"Operation cannot be fulfilled on rayjobs.ray.io \"fail\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-11-03T15:03:12.733Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail","namespace":"test-ns-g9m7p"},"reconcileID":"ace119fe-2c7a-4e17-beaa-2762ee18a926"}
{"level":"error","ts":"2026-11-03T15:03:12.733Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail","namespace":"test-ns-g9m7p"},"reconcileID":"ace119fe-2c7a-4e17-beaa-2762ee18a926","error":"Operation cannot be fulfilled on rayjobs.ray.io \"fail\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:03:12.733Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-g9m7p"},"reconcileID":"58e74612-0e32-4e71-8809-7c69d50e09e8","JobStatus":"FAILED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:03:12.733Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-g9m7p"},"reconcileID":"58e74612-0e32-4e71-8809-7c69d50e09e8","name":"raycluster"}
{"level":"info","ts":"2026-11-03T15:03:12.733Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-g9m7p"},"reconcileID":"58e74612-0e32-4e71-8809-7c69d50e09e8","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"}}
{"level":"error","ts":"2026-11-03T15:03:12.736Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-g9m7p"},"reconcileID":"58e74612-0e32-4e71-8809-7c69d50e09e8","JobId":"fail-bbpjq","error":"Get \"http://raycluster-head-svc.test-ns-g9m7p.svc.cluster.local:8265/api/jobs/fail-bbpjq\": dial tcp 10.132.0.31:8265: connect: connection refused","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:321\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:03:12.854Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"7e7394ba-78c5-4f40-9c02-5b516a4b4dda"}
{"level":"info","ts":"2026-11-03T15:03:12.854Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"7e7394ba-78c5-4f40-9c02-5b516a4b4dda"}
{"level":"info","ts":"2026-11-03T15:03:12.854Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"7e7394ba-78c5-4f40-9c02-5b516a4b4dda","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:03:12.854Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"7e7394ba-78c5-4f40-9c02-5b516a4b4dda","Found 1 head Pod":"raycluster-head-b4s9s","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:03:12.854Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"7e7394ba-78c5-4f40-9c02-5b516a4b4dda","head Pod":"raycluster-head-b4s9s","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-b4s9s. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:03:12.854Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"7e7394ba-78c5-4f40-9c02-5b516a4b4dda","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:03:12.854Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"7e7394ba-78c5-4f40-9c02-5b516a4b4dda","worker Pod":"raycluster-small-group-worker-qrhm7","shouldDelete":true,"reason":"The worker Pod raycluster-small-group-worker-qrhm7 status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary."}
{"level":"info","ts":"2026-11-03T15:03:12.874Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"52f9604b-3b47-4e09-b12c-9117ca6b8911","certificate":"ray-head-cert-counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:03:12.874Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"52f9604b-3b47-4e09-b12c-9117ca6b8911"}
{"level":"error","ts":"2026-11-03T15:03:12.904Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"7e7394ba-78c5-4f40-9c02-5b516a4b4dda","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"delete 1 unhealthy worker Pods","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:03:12.904Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"7e7394ba-78c5-4f40-9c02-5b516a4b4dda","error":"delete 1 unhealthy worker Pods"}
{"level":"info","ts":"2026-11-03T15:03:12.905Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"7e7394ba-78c5-4f40-9c02-5b516a4b4dda","name":"raycluster","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:02:34Z","stateTransitionTimes":{"ready":"2026-11-03T15:02:34Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.31","serviceIP":"10.132.0.31","podName":"raycluster-head-b4s9s","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:02:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:02:34Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T14:58:42Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:12Z","stateTransitionTimes":{"ready":"2026-11-03T15:02:34Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.31","serviceIP":"10.132.0.31","podName":"raycluster-head-b4s9s","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:02:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:02:34Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T14:58:42Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"error","ts":"2026-11-03T15:03:12.954Z","logger":"events","msg":"Server rejected event (will not retry!)","event":{"namespace":"test-ns-g9m7p","name":"raycluster.18e49131633f8c7b"},"error":"events.events.k8s.io \"raycluster.18e49131633f8c7b\" is forbidden: unable to create new content in namespace test-ns-g9m7p because it is being terminated","stacktrace":"k8s.io/client-go/tools/events.recordEvent\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:270\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).attemptRecording\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:211\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).recordToSink.func1\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:200"}
{"level":"info","ts":"2026-11-03T15:03:12.966Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"7e7394ba-78c5-4f40-9c02-5b516a4b4dda"}
{"level":"error","ts":"2026-11-03T15:03:12.966Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"7e7394ba-78c5-4f40-9c02-5b516a4b4dda","error":"delete 1 unhealthy worker Pods","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:03:12.966Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"cee918f2-3889-4741-a4a6-ee1029a98278","cluster":"raycluster"}
{"level":"info","ts":"2026-11-03T15:03:12.972Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"cf29b465-308e-4f82-a994-db45dd868682"}
{"level":"info","ts":"2026-11-03T15:03:12.972Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"cf29b465-308e-4f82-a994-db45dd868682"}
{"level":"info","ts":"2026-11-03T15:03:12.972Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"cf29b465-308e-4f82-a994-db45dd868682","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:03:12.972Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"cf29b465-308e-4f82-a994-db45dd868682","Found 1 head Pod":"raycluster-head-b4s9s","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:03:12.972Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"cf29b465-308e-4f82-a994-db45dd868682","head Pod":"raycluster-head-b4s9s","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-head-b4s9s. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:03:12.972Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"cf29b465-308e-4f82-a994-db45dd868682","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-11-03T15:03:12.972Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"cf29b465-308e-4f82-a994-db45dd868682","podName":"raycluster-small-group-worker-qrhm7","pod Status.Phase":"Failed"}
{"level":"info","ts":"2026-11-03T15:03:12.972Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"cf29b465-308e-4f82-a994-db45dd868682","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:03:12.972Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"cf29b465-308e-4f82-a994-db45dd868682","seconds":300}
{"level":"info","ts":"2026-11-03T15:03:13.194Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"9929407d-7206-43b6-9a0c-d5a1a0615936"}
{"level":"info","ts":"2026-11-03T15:03:13.194Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"9929407d-7206-43b6-9a0c-d5a1a0615936"}
{"level":"info","ts":"2026-11-03T15:03:13.194Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"9929407d-7206-43b6-9a0c-d5a1a0615936","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:03:13.194Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"9929407d-7206-43b6-9a0c-d5a1a0615936","Found 1 head Pod":"raycluster-head-b4s9s","Pod status":"Failed","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"&ContainerStateTerminated{ExitCode:1,Signal:0,Reason:Error,Message:,StartedAt:2026-11-03 15:02:17 +0000 UTC,FinishedAt:2026-11-03 15:03:12 +0000 UTC,ContainerID:cri-o://fa96049559daa2cbde2bb3a4940ee81d5f24aee1e62f9c128386be6d6d4ee253,}"}
{"level":"info","ts":"2026-11-03T15:03:13.194Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"9929407d-7206-43b6-9a0c-d5a1a0615936","head Pod":"raycluster-head-b4s9s","shouldDelete":true,"reason":"The head Pod raycluster-head-b4s9s status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary."}
{"level":"error","ts":"2026-11-03T15:03:13.245Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"9929407d-7206-43b6-9a0c-d5a1a0615936","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"The head Pod raycluster-head-b4s9s status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary.","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:03:13.245Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"9929407d-7206-43b6-9a0c-d5a1a0615936","error":"The head Pod raycluster-head-b4s9s status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary."}
{"level":"info","ts":"2026-11-03T15:03:13.245Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"9929407d-7206-43b6-9a0c-d5a1a0615936","name":"raycluster","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:12Z","stateTransitionTimes":{"ready":"2026-11-03T15:02:34Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.31","serviceIP":"10.132.0.31","podName":"raycluster-head-b4s9s","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:02:32Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:02:34Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T14:58:42Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:13Z","stateTransitionTimes":{"ready":"2026-11-03T15:02:34Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.31","serviceIP":"10.132.0.31","podName":"raycluster-head-b4s9s","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:13Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:02:34Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T14:58:42Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:03:13.270Z","logger":"controllers.RayClusterMTLS","msg":"Updating head certificate","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"cee918f2-3889-4741-a4a6-ee1029a98278","certificate":"ray-head-cert-raycluster","ipAddresses":["127.0.0.1"]}
{"level":"error","ts":"2026-11-03T15:03:13.294Z","logger":"events","msg":"Server rejected event (will not retry!)","event":{"namespace":"test-ns-g9m7p","name":"raycluster.18e491317788b867"},"error":"events.events.k8s.io \"raycluster.18e491317788b867\" is forbidden: unable to create new content in namespace test-ns-g9m7p because it is being terminated","stacktrace":"k8s.io/client-go/tools/events.recordEvent\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:270\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).attemptRecording\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:211\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).recordToSink.func1\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:200"}
{"level":"info","ts":"2026-11-03T15:03:13.306Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"9929407d-7206-43b6-9a0c-d5a1a0615936"}
{"level":"error","ts":"2026-11-03T15:03:13.306Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"9929407d-7206-43b6-9a0c-d5a1a0615936","error":"The head Pod raycluster-head-b4s9s status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary.","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:03:13.311Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"b546f437-8e83-46c3-bdef-f1cc394a39f2"}
{"level":"info","ts":"2026-11-03T15:03:13.311Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"b546f437-8e83-46c3-bdef-f1cc394a39f2"}
{"level":"info","ts":"2026-11-03T15:03:13.311Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"b546f437-8e83-46c3-bdef-f1cc394a39f2","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:03:13.311Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"b546f437-8e83-46c3-bdef-f1cc394a39f2","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-11-03T15:03:13.311Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"b546f437-8e83-46c3-bdef-f1cc394a39f2","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-11-03T15:03:13.311Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"b546f437-8e83-46c3-bdef-f1cc394a39f2","podName":"raycluster-head-b4s9s","pod Status.Phase":"Failed"}
{"level":"info","ts":"2026-11-03T15:03:13.312Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"b546f437-8e83-46c3-bdef-f1cc394a39f2","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:03:13.312Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"b546f437-8e83-46c3-bdef-f1cc394a39f2","seconds":300}
{"level":"error","ts":"2026-11-03T15:03:13.320Z","logger":"events","msg":"Server rejected event (will not retry!)","event":{"namespace":"test-ns-g9m7p","name":"raycluster.18e491317910fd0f"},"error":"events.events.k8s.io \"raycluster.18e491317910fd0f\" is forbidden: unable to create new content in namespace test-ns-g9m7p because it is being terminated","stacktrace":"k8s.io/client-go/tools/events.recordEvent\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:270\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).attemptRecording\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:211\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).recordToSink.func1\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:200"}
{"level":"info","ts":"2026-11-03T15:03:13.530Z","logger":"controllers.RayClusterMTLS","msg":"Updating worker certificate","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"cee918f2-3889-4741-a4a6-ee1029a98278","certificate":"ray-worker-cert-raycluster","ipAddresses":["127.0.0.1"]}
{"level":"info","ts":"2026-11-03T15:03:13.620Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-g9m7p"},"reconcileID":"c2e95dfc-5d2a-431b-9ce5-78a84324508d","JobStatus":"FAILED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:03:13.620Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-g9m7p"},"reconcileID":"c2e95dfc-5d2a-431b-9ce5-78a84324508d","name":"raycluster"}
{"level":"info","ts":"2026-11-03T15:03:13.620Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-g9m7p"},"reconcileID":"c2e95dfc-5d2a-431b-9ce5-78a84324508d","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"}}
{"level":"error","ts":"2026-11-03T15:03:13.630Z","logger":"events","msg":"Server rejected event (will not retry!)","event":{"namespace":"test-ns-g9m7p","name":"raycluster.18e491317910fd0f"},"error":"events.events.k8s.io \"raycluster.18e491317910fd0f\" is forbidden: unable to create new content in namespace test-ns-g9m7p because it is being terminated","stacktrace":"k8s.io/client-go/tools/events.recordEvent\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:270\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).attemptRecording\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:211\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).recordToSink.func1\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:200"}
{"level":"info","ts":"2026-11-03T15:03:13.661Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ad64df02-7f21-4ea4-ad2b-2bf298650c25"}
{"level":"info","ts":"2026-11-03T15:03:13.661Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ad64df02-7f21-4ea4-ad2b-2bf298650c25"}
{"level":"info","ts":"2026-11-03T15:03:13.661Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ad64df02-7f21-4ea4-ad2b-2bf298650c25","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:03:13.661Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ad64df02-7f21-4ea4-ad2b-2bf298650c25","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-11-03T15:03:13.661Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ad64df02-7f21-4ea4-ad2b-2bf298650c25","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-11-03T15:03:13.662Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ad64df02-7f21-4ea4-ad2b-2bf298650c25","podName":"raycluster-head-b4s9s","pod Status.Phase":"Failed"}
{"level":"info","ts":"2026-11-03T15:03:13.662Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ad64df02-7f21-4ea4-ad2b-2bf298650c25","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:03:13.662Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ad64df02-7f21-4ea4-ad2b-2bf298650c25","seconds":300}
{"level":"info","ts":"2026-11-03T15:03:13.665Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"c1d36957-0b94-4401-97ea-947c23cdf3cb"}
{"level":"info","ts":"2026-11-03T15:03:13.665Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"c1d36957-0b94-4401-97ea-947c23cdf3cb"}
{"level":"info","ts":"2026-11-03T15:03:13.665Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"c1d36957-0b94-4401-97ea-947c23cdf3cb","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:03:13.666Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"c1d36957-0b94-4401-97ea-947c23cdf3cb","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-11-03T15:03:13.666Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"c1d36957-0b94-4401-97ea-947c23cdf3cb","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:03:13.666Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"c1d36957-0b94-4401-97ea-947c23cdf3cb","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:03:13.666Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"c1d36957-0b94-4401-97ea-947c23cdf3cb","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-11-03T15:03:13.666Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"c1d36957-0b94-4401-97ea-947c23cdf3cb","group":"small-group","indices":{}}
{"level":"info","ts":"2026-11-03T15:03:13.666Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"c1d36957-0b94-4401-97ea-947c23cdf3cb","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-11-03T15:03:13.716Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before worker pod creation","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"c1d36957-0b94-4401-97ea-947c23cdf3cb","error":"certificate ray-head-cert-raycluster is not ready at current generation 3"}
{"level":"error","ts":"2026-11-03T15:03:13.716Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"c1d36957-0b94-4401-97ea-947c23cdf3cb","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster is not ready at current generation 3","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:03:13.716Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"c1d36957-0b94-4401-97ea-947c23cdf3cb","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster is not ready at current generation 3"}
{"level":"info","ts":"2026-11-03T15:03:13.716Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"c1d36957-0b94-4401-97ea-947c23cdf3cb"}
{"level":"error","ts":"2026-11-03T15:03:13.716Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"c1d36957-0b94-4401-97ea-947c23cdf3cb","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster is not ready at current generation 3","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:03:13.721Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"9cd2f65e-d368-405f-bf3c-34bc23103454"}
{"level":"info","ts":"2026-11-03T15:03:13.721Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"9cd2f65e-d368-405f-bf3c-34bc23103454"}
{"level":"info","ts":"2026-11-03T15:03:13.721Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"9cd2f65e-d368-405f-bf3c-34bc23103454","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:03:13.722Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"9cd2f65e-d368-405f-bf3c-34bc23103454","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-11-03T15:03:13.722Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"9cd2f65e-d368-405f-bf3c-34bc23103454","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:03:13.722Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"9cd2f65e-d368-405f-bf3c-34bc23103454","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:03:13.722Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"9cd2f65e-d368-405f-bf3c-34bc23103454","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-11-03T15:03:13.722Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"9cd2f65e-d368-405f-bf3c-34bc23103454","group":"small-group","indices":{}}
{"level":"info","ts":"2026-11-03T15:03:13.722Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"9cd2f65e-d368-405f-bf3c-34bc23103454","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-11-03T15:03:13.772Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before worker pod creation","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"9cd2f65e-d368-405f-bf3c-34bc23103454","error":"certificate ray-head-cert-raycluster is not ready at current generation 3"}
{"level":"error","ts":"2026-11-03T15:03:13.772Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"9cd2f65e-d368-405f-bf3c-34bc23103454","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster is not ready at current generation 3","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:03:13.772Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"9cd2f65e-d368-405f-bf3c-34bc23103454","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster is not ready at current generation 3"}
{"level":"info","ts":"2026-11-03T15:03:13.772Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"9cd2f65e-d368-405f-bf3c-34bc23103454"}
{"level":"error","ts":"2026-11-03T15:03:13.772Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"9cd2f65e-d368-405f-bf3c-34bc23103454","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster is not ready at current generation 3","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:03:13.777Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"f96a5289-d958-4e57-8cc4-62079d8034ae"}
{"level":"info","ts":"2026-11-03T15:03:13.777Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"f96a5289-d958-4e57-8cc4-62079d8034ae"}
{"level":"info","ts":"2026-11-03T15:03:13.777Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"f96a5289-d958-4e57-8cc4-62079d8034ae","1 head service found":"counter-5gq9h-head-svc"}
{"level":"info","ts":"2026-11-03T15:03:13.778Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"f96a5289-d958-4e57-8cc4-62079d8034ae","Found 1 head Pod":"counter-5gq9h-head-9twx4","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:03:13.778Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"f96a5289-d958-4e57-8cc4-62079d8034ae","head Pod":"counter-5gq9h-head-9twx4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-5gq9h-head-9twx4. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:03:13.778Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"f96a5289-d958-4e57-8cc4-62079d8034ae","podName":"counter-5gq9h-head-9twx4","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:03:13.778Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"f96a5289-d958-4e57-8cc4-62079d8034ae","podName":"counter-5gq9h-head-9twx4","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:03:13.778Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"f96a5289-d958-4e57-8cc4-62079d8034ae","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:03:13.778Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"f96a5289-d958-4e57-8cc4-62079d8034ae","seconds":300}
{"level":"info","ts":"2026-11-03T15:03:13.783Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"79530642-b3fb-45ff-96b2-70a8e5758016"}
{"level":"info","ts":"2026-11-03T15:03:13.783Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"79530642-b3fb-45ff-96b2-70a8e5758016"}
{"level":"info","ts":"2026-11-03T15:03:13.783Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"79530642-b3fb-45ff-96b2-70a8e5758016","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:03:13.783Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"79530642-b3fb-45ff-96b2-70a8e5758016","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-11-03T15:03:13.783Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"79530642-b3fb-45ff-96b2-70a8e5758016","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:03:13.783Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"79530642-b3fb-45ff-96b2-70a8e5758016","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:03:13.783Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"79530642-b3fb-45ff-96b2-70a8e5758016","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-11-03T15:03:13.783Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"79530642-b3fb-45ff-96b2-70a8e5758016","group":"small-group","indices":{}}
{"level":"info","ts":"2026-11-03T15:03:13.783Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"79530642-b3fb-45ff-96b2-70a8e5758016","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-11-03T15:03:13.787Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"cee918f2-3889-4741-a4a6-ee1029a98278","certificate":"ray-head-cert-raycluster"}
{"level":"info","ts":"2026-11-03T15:03:13.787Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"cee918f2-3889-4741-a4a6-ee1029a98278"}
{"level":"info","ts":"2026-11-03T15:03:13.787Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"e4a2e1d3-4b88-4810-ab50-cdf9e6871b82","cluster":"raycluster"}
{"level":"info","ts":"2026-11-03T15:03:13.833Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before worker pod creation","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"79530642-b3fb-45ff-96b2-70a8e5758016","error":"certificate ray-head-cert-raycluster is not ready at current generation 3"}
{"level":"error","ts":"2026-11-03T15:03:13.833Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"79530642-b3fb-45ff-96b2-70a8e5758016","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster is not ready at current generation 3","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:03:13.833Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"79530642-b3fb-45ff-96b2-70a8e5758016","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster is not ready at current generation 3"}
{"level":"info","ts":"2026-11-03T15:03:13.833Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"79530642-b3fb-45ff-96b2-70a8e5758016"}
{"level":"error","ts":"2026-11-03T15:03:13.833Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"79530642-b3fb-45ff-96b2-70a8e5758016","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster is not ready at current generation 3","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-03T15:03:13.835Z","logger":"events","msg":"Server rejected event (will not retry!)","event":{"namespace":"test-ns-g9m7p","name":"raycluster.18e4913197d5d641"},"error":"events.events.k8s.io \"raycluster.18e4913197d5d641\" is forbidden: unable to create new content in namespace test-ns-g9m7p because it is being terminated","stacktrace":"k8s.io/client-go/tools/events.recordEvent\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:270\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).attemptRecording\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:211\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).recordToSink.func1\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:200"}
{"level":"info","ts":"2026-11-03T15:03:13.854Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"d1ed8799-5b5b-4084-8135-b4d6a75b872b"}
{"level":"info","ts":"2026-11-03T15:03:13.854Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"d1ed8799-5b5b-4084-8135-b4d6a75b872b"}
{"level":"info","ts":"2026-11-03T15:03:13.854Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"d1ed8799-5b5b-4084-8135-b4d6a75b872b","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:03:13.854Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"d1ed8799-5b5b-4084-8135-b4d6a75b872b","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-11-03T15:03:13.854Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"d1ed8799-5b5b-4084-8135-b4d6a75b872b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:03:13.854Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"d1ed8799-5b5b-4084-8135-b4d6a75b872b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:03:13.854Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"d1ed8799-5b5b-4084-8135-b4d6a75b872b","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-11-03T15:03:13.854Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"d1ed8799-5b5b-4084-8135-b4d6a75b872b","group":"small-group","indices":{}}
{"level":"info","ts":"2026-11-03T15:03:13.854Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"d1ed8799-5b5b-4084-8135-b4d6a75b872b","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-11-03T15:03:13.905Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before worker pod creation","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"d1ed8799-5b5b-4084-8135-b4d6a75b872b","error":"certificate ray-head-cert-raycluster is not ready at current generation 3"}
{"level":"error","ts":"2026-11-03T15:03:13.905Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"d1ed8799-5b5b-4084-8135-b4d6a75b872b","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster is not ready at current generation 3","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:03:13.905Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"d1ed8799-5b5b-4084-8135-b4d6a75b872b","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster is not ready at current generation 3"}
{"level":"info","ts":"2026-11-03T15:03:13.905Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"d1ed8799-5b5b-4084-8135-b4d6a75b872b"}
{"level":"error","ts":"2026-11-03T15:03:13.905Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"d1ed8799-5b5b-4084-8135-b4d6a75b872b","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster is not ready at current generation 3","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:03:13.946Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"d1100417-dcda-44ad-8c12-4ffe30b13352"}
{"level":"info","ts":"2026-11-03T15:03:13.946Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"d1100417-dcda-44ad-8c12-4ffe30b13352"}
{"level":"info","ts":"2026-11-03T15:03:13.946Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"d1100417-dcda-44ad-8c12-4ffe30b13352","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:03:13.946Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"d1100417-dcda-44ad-8c12-4ffe30b13352","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-11-03T15:03:13.946Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"d1100417-dcda-44ad-8c12-4ffe30b13352","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:03:13.946Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"d1100417-dcda-44ad-8c12-4ffe30b13352","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:03:13.946Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"d1100417-dcda-44ad-8c12-4ffe30b13352","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-11-03T15:03:13.946Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"d1100417-dcda-44ad-8c12-4ffe30b13352","group":"small-group","indices":{}}
{"level":"info","ts":"2026-11-03T15:03:13.946Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"d1100417-dcda-44ad-8c12-4ffe30b13352","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-11-03T15:03:13.996Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before worker pod creation","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"d1100417-dcda-44ad-8c12-4ffe30b13352","error":"certificate ray-head-cert-raycluster is not ready at current generation 3"}
{"level":"error","ts":"2026-11-03T15:03:13.996Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"d1100417-dcda-44ad-8c12-4ffe30b13352","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster is not ready at current generation 3","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:03:13.996Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"d1100417-dcda-44ad-8c12-4ffe30b13352","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster is not ready at current generation 3"}
{"level":"info","ts":"2026-11-03T15:03:13.996Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"d1100417-dcda-44ad-8c12-4ffe30b13352"}
{"level":"error","ts":"2026-11-03T15:03:13.996Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"d1100417-dcda-44ad-8c12-4ffe30b13352","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster is not ready at current generation 3","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:03:14.028Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"0c602a02-7554-4ae5-863b-079486f43875"}
{"level":"info","ts":"2026-11-03T15:03:14.028Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"0c602a02-7554-4ae5-863b-079486f43875"}
{"level":"info","ts":"2026-11-03T15:03:14.028Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"0c602a02-7554-4ae5-863b-079486f43875","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:03:14.028Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"0c602a02-7554-4ae5-863b-079486f43875","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-11-03T15:03:14.028Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"0c602a02-7554-4ae5-863b-079486f43875","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:03:14.028Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"0c602a02-7554-4ae5-863b-079486f43875","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:03:14.028Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"0c602a02-7554-4ae5-863b-079486f43875","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-11-03T15:03:14.028Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"0c602a02-7554-4ae5-863b-079486f43875","group":"small-group","indices":{}}
{"level":"info","ts":"2026-11-03T15:03:14.028Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"0c602a02-7554-4ae5-863b-079486f43875","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-11-03T15:03:14.078Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before worker pod creation","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"0c602a02-7554-4ae5-863b-079486f43875","error":"certificate ray-head-cert-raycluster is not ready at current generation 3"}
{"level":"error","ts":"2026-11-03T15:03:14.078Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"0c602a02-7554-4ae5-863b-079486f43875","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster is not ready at current generation 3","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:03:14.078Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"0c602a02-7554-4ae5-863b-079486f43875","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster is not ready at current generation 3"}
{"level":"info","ts":"2026-11-03T15:03:14.078Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"0c602a02-7554-4ae5-863b-079486f43875","filter labels":["test-ns-g9m7p",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:03:14.078Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"0c602a02-7554-4ae5-863b-079486f43875","filter labels":["test-ns-g9m7p",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:03:14.078Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"0c602a02-7554-4ae5-863b-079486f43875","filter labels":["test-ns-g9m7p",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:03:14.078Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"0c602a02-7554-4ae5-863b-079486f43875","name":"raycluster","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:13Z","stateTransitionTimes":{"ready":"2026-11-03T15:02:34Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.31","serviceIP":"10.132.0.31","podName":"raycluster-head-b4s9s","serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:13Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:02:34Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T14:58:42Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:14Z","stateTransitionTimes":{"ready":"2026-11-03T15:02:34Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:13Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:02:34Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T14:58:41Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T14:58:42Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:03:14.139Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"0c602a02-7554-4ae5-863b-079486f43875"}
{"level":"error","ts":"2026-11-03T15:03:14.139Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"0c602a02-7554-4ae5-863b-079486f43875","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster is not ready at current generation 3","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:03:14.139Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"2466dffb-a35c-4728-9ab1-d5f311d57971"}
{"level":"info","ts":"2026-11-03T15:03:14.139Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"2466dffb-a35c-4728-9ab1-d5f311d57971"}
{"level":"info","ts":"2026-11-03T15:03:14.139Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"2466dffb-a35c-4728-9ab1-d5f311d57971","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:03:14.139Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"2466dffb-a35c-4728-9ab1-d5f311d57971"}
{"level":"info","ts":"2026-11-03T15:03:14.189Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"2466dffb-a35c-4728-9ab1-d5f311d57971","error":"certificate ray-head-cert-raycluster is not ready at current generation 3"}
{"level":"error","ts":"2026-11-03T15:03:14.189Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"2466dffb-a35c-4728-9ab1-d5f311d57971","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster is not ready at current generation 3","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:03:14.189Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"2466dffb-a35c-4728-9ab1-d5f311d57971","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster is not ready at current generation 3"}
{"level":"info","ts":"2026-11-03T15:03:14.189Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"2466dffb-a35c-4728-9ab1-d5f311d57971","filter labels":["test-ns-g9m7p",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:03:14.189Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"2466dffb-a35c-4728-9ab1-d5f311d57971","filter labels":["test-ns-g9m7p",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:03:14.189Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"2466dffb-a35c-4728-9ab1-d5f311d57971","filter labels":["test-ns-g9m7p",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:03:14.189Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"2466dffb-a35c-4728-9ab1-d5f311d57971"}
{"level":"error","ts":"2026-11-03T15:03:14.189Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"2466dffb-a35c-4728-9ab1-d5f311d57971","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster is not ready at current generation 3","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:03:14.392Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"e4a2e1d3-4b88-4810-ab50-cdf9e6871b82","certificate":"ray-head-cert-raycluster"}
{"level":"info","ts":"2026-11-03T15:03:14.392Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"e4a2e1d3-4b88-4810-ab50-cdf9e6871b82"}
{"level":"info","ts":"2026-11-03T15:03:14.392Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"1b141c15-76db-4cd1-8136-5d25b2419219","cluster":"raycluster"}
{"level":"error","ts":"2026-11-03T15:03:14.441Z","logger":"events","msg":"Server rejected event (will not retry!)","event":{"namespace":"test-ns-g9m7p","name":"raycluster.18e49131bbec02c2"},"error":"events.events.k8s.io \"raycluster.18e49131bbec02c2\" is forbidden: unable to create new content in namespace test-ns-g9m7p because it is being terminated","stacktrace":"k8s.io/client-go/tools/events.recordEvent\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:270\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).attemptRecording\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:211\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).recordToSink.func1\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:200"}
{"level":"info","ts":"2026-11-03T15:03:14.511Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"9de2ecb5-e348-4472-ad1a-bfff87579a1d"}
{"level":"info","ts":"2026-11-03T15:03:14.511Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"9de2ecb5-e348-4472-ad1a-bfff87579a1d"}
{"level":"info","ts":"2026-11-03T15:03:14.511Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"9de2ecb5-e348-4472-ad1a-bfff87579a1d","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:03:14.511Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"9de2ecb5-e348-4472-ad1a-bfff87579a1d"}
{"level":"info","ts":"2026-11-03T15:03:14.562Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"9de2ecb5-e348-4472-ad1a-bfff87579a1d","error":"certificate ray-head-cert-raycluster is not ready at current generation 3"}
{"level":"error","ts":"2026-11-03T15:03:14.562Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"9de2ecb5-e348-4472-ad1a-bfff87579a1d","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster is not ready at current generation 3","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:03:14.562Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"9de2ecb5-e348-4472-ad1a-bfff87579a1d","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster is not ready at current generation 3"}
{"level":"info","ts":"2026-11-03T15:03:14.562Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"9de2ecb5-e348-4472-ad1a-bfff87579a1d","filter labels":["test-ns-g9m7p",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:03:14.562Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"9de2ecb5-e348-4472-ad1a-bfff87579a1d","filter labels":["test-ns-g9m7p",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:03:14.562Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"9de2ecb5-e348-4472-ad1a-bfff87579a1d","filter labels":["test-ns-g9m7p",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:03:14.562Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"9de2ecb5-e348-4472-ad1a-bfff87579a1d"}
{"level":"error","ts":"2026-11-03T15:03:14.562Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"9de2ecb5-e348-4472-ad1a-bfff87579a1d","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster is not ready at current generation 3","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:03:15.000Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"1b141c15-76db-4cd1-8136-5d25b2419219","certificate":"ray-head-cert-raycluster"}
{"level":"info","ts":"2026-11-03T15:03:15.000Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"1b141c15-76db-4cd1-8136-5d25b2419219"}
{"level":"error","ts":"2026-11-03T15:03:15.049Z","logger":"events","msg":"Server rejected event (will not retry!)","event":{"namespace":"test-ns-g9m7p","name":"raycluster.18e49131e028e97b"},"error":"events.events.k8s.io \"raycluster.18e49131e028e97b\" is forbidden: unable to create new content in namespace test-ns-g9m7p because it is being terminated","stacktrace":"k8s.io/client-go/tools/events.recordEvent\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:270\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).attemptRecording\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:211\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).recordToSink.func1\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:200"}
{"level":"info","ts":"2026-11-03T15:03:15.202Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ff1d338a-5260-4cb7-b1f7-cb475a6504d5"}
{"level":"info","ts":"2026-11-03T15:03:15.202Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ff1d338a-5260-4cb7-b1f7-cb475a6504d5"}
{"level":"info","ts":"2026-11-03T15:03:15.202Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ff1d338a-5260-4cb7-b1f7-cb475a6504d5","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:03:15.202Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ff1d338a-5260-4cb7-b1f7-cb475a6504d5"}
{"level":"info","ts":"2026-11-03T15:03:15.253Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ff1d338a-5260-4cb7-b1f7-cb475a6504d5","error":"certificate ray-head-cert-raycluster is not ready at current generation 3"}
{"level":"error","ts":"2026-11-03T15:03:15.253Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ff1d338a-5260-4cb7-b1f7-cb475a6504d5","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster is not ready at current generation 3","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:03:15.253Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ff1d338a-5260-4cb7-b1f7-cb475a6504d5","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster is not ready at current generation 3"}
{"level":"info","ts":"2026-11-03T15:03:15.253Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ff1d338a-5260-4cb7-b1f7-cb475a6504d5","filter labels":["test-ns-g9m7p",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:03:15.253Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ff1d338a-5260-4cb7-b1f7-cb475a6504d5","filter labels":["test-ns-g9m7p",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:03:15.253Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ff1d338a-5260-4cb7-b1f7-cb475a6504d5","filter labels":["test-ns-g9m7p",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:03:15.253Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ff1d338a-5260-4cb7-b1f7-cb475a6504d5"}
{"level":"error","ts":"2026-11-03T15:03:15.253Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ff1d338a-5260-4cb7-b1f7-cb475a6504d5","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster is not ready at current generation 3","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-03T15:03:15.621Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"fail","namespace":"test-ns-g9m7p"},"reconcileID":"c2e95dfc-5d2a-431b-9ce5-78a84324508d","JobId":"fail-bbpjq","error":"Get \"http://raycluster-head-svc.test-ns-g9m7p.svc.cluster.local:8265/api/jobs/fail-bbpjq\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:321\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:03:15.621Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"d1b489fe-d597-4531-95fc-478dcdf2349e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:03:15.621Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"d1b489fe-d597-4531-95fc-478dcdf2349e","name":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:03:15.621Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"d1b489fe-d597-4531-95fc-478dcdf2349e","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:03:15.621Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"d1b489fe-d597-4531-95fc-478dcdf2349e"}
{"level":"info","ts":"2026-11-03T15:03:15.621Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"d1b489fe-d597-4531-95fc-478dcdf2349e","RayCluster":"counter-5gq9h","State":""}
{"level":"info","ts":"2026-11-03T15:03:15.621Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"d1b489fe-d597-4531-95fc-478dcdf2349e","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:03:16.534Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"5fb24860-8ced-4e3d-82dc-8334faaa1b2b"}
{"level":"info","ts":"2026-11-03T15:03:16.534Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"5fb24860-8ced-4e3d-82dc-8334faaa1b2b"}
{"level":"info","ts":"2026-11-03T15:03:16.534Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"5fb24860-8ced-4e3d-82dc-8334faaa1b2b","1 head service found":"raycluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:03:16.534Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"5fb24860-8ced-4e3d-82dc-8334faaa1b2b"}
{"level":"info","ts":"2026-11-03T15:03:16.584Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"5fb24860-8ced-4e3d-82dc-8334faaa1b2b","error":"certificate ray-head-cert-raycluster is not ready at current generation 3"}
{"level":"error","ts":"2026-11-03T15:03:16.584Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"5fb24860-8ced-4e3d-82dc-8334faaa1b2b","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster is not ready at current generation 3","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:03:16.584Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"5fb24860-8ced-4e3d-82dc-8334faaa1b2b","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster is not ready at current generation 3"}
{"level":"info","ts":"2026-11-03T15:03:16.585Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"5fb24860-8ced-4e3d-82dc-8334faaa1b2b","filter labels":["test-ns-g9m7p",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:03:16.585Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"5fb24860-8ced-4e3d-82dc-8334faaa1b2b","filter labels":["test-ns-g9m7p",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:03:16.585Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"5fb24860-8ced-4e3d-82dc-8334faaa1b2b","filter labels":["test-ns-g9m7p",{"ray.io/cluster":"raycluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:03:16.585Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"5fb24860-8ced-4e3d-82dc-8334faaa1b2b"}
{"level":"error","ts":"2026-11-03T15:03:16.585Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"5fb24860-8ced-4e3d-82dc-8334faaa1b2b","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster is not ready at current generation 3","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:03:16.952Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"counter","namespace":"test-ns-g9m7p"},"reconcileID":"b79ea57d-8a95-476c-ac60-5c600f80bd6b","DeletionTimestamp":"2026-11-03 15:03:16 +0000 UTC"}
{"level":"info","ts":"2026-11-03T15:03:16.952Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"counter","namespace":"test-ns-g9m7p"},"reconcileID":"b79ea57d-8a95-476c-ac60-5c600f80bd6b","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-11-03T15:03:16.979Z","logger":"controllers.NetworkPolicy","msg":"Reconciling NetworkPolicies for RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"de96ae45-6817-4925-b7ef-1f2b1d3e4555","cluster":"raycluster","namespace":"test-ns-g9m7p"}
{"level":"info","ts":"2026-11-03T15:03:17.011Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"fail","namespace":"test-ns-g9m7p"},"reconcileID":"4322f3d8-cba1-468f-a100-65c766b177a0","DeletionTimestamp":"2026-11-03 15:03:16 +0000 UTC"}
{"level":"info","ts":"2026-11-03T15:03:17.011Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"fail","namespace":"test-ns-g9m7p"},"reconcileID":"4322f3d8-cba1-468f-a100-65c766b177a0","finalizer":"ray.io/rayjob-finalizer"}
{"level":"error","ts":"2026-11-03T15:03:17.028Z","logger":"controllers.NetworkPolicy","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"de96ae45-6817-4925-b7ef-1f2b1d3e4555","error":"networkpolicies.networking.k8s.io \"raycluster-head\" is forbidden: unable to create new content in namespace test-ns-g9m7p because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:03:17.028Z","logger":"controllers.NetworkPolicy","msg":"Reconciling NetworkPolicies for RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"f28347c0-a073-49ea-9fd1-16965c9e01d3","cluster":"raycluster","namespace":"test-ns-g9m7p"}
{"level":"info","ts":"2026-11-03T15:03:17.072Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"managed-externally","namespace":"test-ns-g9m7p"},"reconcileID":"f69d631d-4573-4de0-bbc5-09db2602952f"}
{"level":"info","ts":"2026-11-03T15:03:17.072Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"counter","namespace":"test-ns-g9m7p"},"reconcileID":"b9024b62-1c57-4a42-b0d6-a1072d38cfb1"}
{"level":"info","ts":"2026-11-03T15:03:17.072Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"fail","namespace":"test-ns-g9m7p"},"reconcileID":"56773ed1-e476-4189-9ca4-70345ebfde14"}
{"level":"error","ts":"2026-11-03T15:03:17.078Z","logger":"events","msg":"Server rejected event (will not retry!)","event":{"namespace":"test-ns-g9m7p","name":"raycluster.18e49132590e1510"},"error":"events.events.k8s.io \"raycluster.18e49132590e1510\" is forbidden: unable to create new content in namespace test-ns-g9m7p because it is being terminated","stacktrace":"k8s.io/client-go/tools/events.recordEvent\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:270\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).attemptRecording\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:211\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).recordToSink.func1\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:200"}
{"level":"error","ts":"2026-11-03T15:03:17.078Z","logger":"controllers.NetworkPolicy","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"f28347c0-a073-49ea-9fd1-16965c9e01d3","error":"networkpolicies.networking.k8s.io \"raycluster-head\" is forbidden: unable to create new content in namespace test-ns-g9m7p because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:03:17.079Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"fail","namespace":"test-ns-g9m7p"},"reconcileID":"5a6f98ef-ab4e-48fb-8d00-6a4f6f4cb444"}
{"level":"info","ts":"2026-11-03T15:03:17.089Z","logger":"controllers.NetworkPolicy","msg":"Reconciling NetworkPolicies for RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ccb0b73a-c8d4-422c-b05e-e7c942dc03c8","cluster":"raycluster","namespace":"test-ns-g9m7p"}
{"level":"info","ts":"2026-11-03T15:03:17.118Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"namespace":"test-ns-g9m7p","name":"raycluster","reconcileID":"fb526db7-1b1b-4779-b6bc-fed64b504947","namespacedName":{"name":"raycluster","namespace":"test-ns-g9m7p"}}
{"level":"info","ts":"2026-11-03T15:03:17.118Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"namespace":"test-ns-g9m7p","name":"raycluster","reconcileID":"fb526db7-1b1b-4779-b6bc-fed64b504947","cluster":"raycluster"}
{"level":"info","ts":"2026-11-03T15:03:17.118Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"namespace":"test-ns-g9m7p","name":"raycluster","reconcileID":"fb526db7-1b1b-4779-b6bc-fed64b504947","cluster":"raycluster"}
{"level":"info","ts":"2026-11-03T15:03:17.118Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"d7c6c219-0a05-4595-879f-0fd7a60eb893"}
{"level":"info","ts":"2026-11-03T15:03:17.118Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"namespace":"test-ns-g9m7p","name":"raycluster","reconcileID":"fb526db7-1b1b-4779-b6bc-fed64b504947","namespace":"test-ns-g9m7p","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-11-03T15:03:17.118Z","logger":"authentication-controller","msg":"ReferenceGrant already deleted","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"namespace":"test-ns-g9m7p","name":"raycluster","reconcileID":"fb526db7-1b1b-4779-b6bc-fed64b504947","name":"kuberay-gateway-access"}
{"level":"error","ts":"2026-11-03T15:03:17.138Z","logger":"controllers.NetworkPolicy","msg":"Reconciler error","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"reconcileID":"ccb0b73a-c8d4-422c-b05e-e7c942dc03c8","error":"networkpolicies.networking.k8s.io \"raycluster-head\" is forbidden: unable to create new content in namespace test-ns-g9m7p because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:03:17.171Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"namespace":"test-ns-g9m7p","name":"raycluster","reconcileID":"fb526db7-1b1b-4779-b6bc-fed64b504947","name":"test-ns-g9m7p-raycluster","namespace":"opendatahub"}
{"level":"error","ts":"2026-11-03T15:03:17.177Z","logger":"events","msg":"Server rejected event (will not retry!)","event":{"namespace":"test-ns-g9m7p","name":"raycluster.18e49132590e1510"},"error":"events.events.k8s.io \"raycluster.18e49132590e1510\" is forbidden: unable to create new content in namespace test-ns-g9m7p because it is being terminated","stacktrace":"k8s.io/client-go/tools/events.recordEvent\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:270\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).attemptRecording\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:211\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).recordToSink.func1\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:200"}
{"level":"info","ts":"2026-11-03T15:03:17.223Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"namespace":"test-ns-g9m7p","name":"raycluster","reconcileID":"fb526db7-1b1b-4779-b6bc-fed64b504947","configMap":"kube-rbac-proxy-config-raycluster"}
{"level":"info","ts":"2026-11-03T15:03:17.277Z","logger":"raycluster-resource","msg":"default","name":"raycluster"}
{"level":"info","ts":"2026-11-03T15:03:17.277Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"raycluster","namespace":"test-ns-g9m7p"}
{"level":"info","ts":"2026-11-03T15:03:17.329Z","logger":"raycluster-resource","msg":"validate update","name":"raycluster"}
{"level":"info","ts":"2026-11-03T15:03:17.386Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"namespace":"test-ns-g9m7p","name":"raycluster","reconcileID":"fb526db7-1b1b-4779-b6bc-fed64b504947","cluster":"raycluster"}
{"level":"info","ts":"2026-11-03T15:03:17.386Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"namespace":"test-ns-g9m7p","name":"raycluster","reconcileID":"e2066cd0-74ba-4623-83e6-c60803be7a9a","namespacedName":{"name":"raycluster","namespace":"test-ns-g9m7p"}}
{"level":"info","ts":"2026-11-03T15:03:17.386Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"namespace":"test-ns-g9m7p","name":"raycluster","reconcileID":"e2066cd0-74ba-4623-83e6-c60803be7a9a","cluster":"raycluster"}
{"level":"info","ts":"2026-11-03T15:03:17.386Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"namespace":"test-ns-g9m7p","name":"raycluster","reconcileID":"e2066cd0-74ba-4623-83e6-c60803be7a9a","cluster":"raycluster"}
{"level":"info","ts":"2026-11-03T15:03:17.386Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"namespace":"test-ns-g9m7p","name":"raycluster","reconcileID":"e2066cd0-74ba-4623-83e6-c60803be7a9a","namespace":"test-ns-g9m7p","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-11-03T15:03:17.386Z","logger":"authentication-controller","msg":"ReferenceGrant already deleted","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"namespace":"test-ns-g9m7p","name":"raycluster","reconcileID":"e2066cd0-74ba-4623-83e6-c60803be7a9a","name":"kuberay-gateway-access"}
{"level":"error","ts":"2026-11-03T15:03:17.437Z","logger":"authentication-controller","msg":"Failed to remove finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"namespace":"test-ns-g9m7p","name":"raycluster","reconcileID":"e2066cd0-74ba-4623-83e6-c60803be7a9a","error":"rayclusters.ray.io \"raycluster\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleDeletion\n\t/workspace/controllers/ray/authentication_controller.go:195\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:03:17.437Z","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"namespace":"test-ns-g9m7p","name":"raycluster","reconcileID":"e2066cd0-74ba-4623-83e6-c60803be7a9a"}
{"level":"error","ts":"2026-11-03T15:03:17.437Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"namespace":"test-ns-g9m7p","name":"raycluster","reconcileID":"e2066cd0-74ba-4623-83e6-c60803be7a9a","error":"rayclusters.ray.io \"raycluster\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:03:17.437Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"namespace":"test-ns-g9m7p","name":"raycluster","reconcileID":"aaabe904-22f4-4521-9028-561d3c65cfc2","namespacedName":{"name":"raycluster","namespace":"test-ns-g9m7p"}}
{"level":"info","ts":"2026-11-03T15:03:17.437Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"namespace":"test-ns-g9m7p","name":"raycluster","reconcileID":"aaabe904-22f4-4521-9028-561d3c65cfc2"}
{"level":"info","ts":"2026-11-03T15:03:17.437Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"namespace":"test-ns-g9m7p","name":"raycluster","reconcileID":"aaabe904-22f4-4521-9028-561d3c65cfc2","namespace":"test-ns-g9m7p","authClusters":0}
{"level":"info","ts":"2026-11-03T15:03:17.437Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster","namespace":"test-ns-g9m7p"},"namespace":"test-ns-g9m7p","name":"raycluster","reconcileID":"aaabe904-22f4-4521-9028-561d3c65cfc2","namespace":"test-ns-g9m7p"}
{"level":"info","ts":"2026-11-03T15:03:18.622Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"97178669-11bd-4bc1-a65e-4d92202b8a83","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:03:18.622Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"97178669-11bd-4bc1-a65e-4d92202b8a83","name":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:03:18.622Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"97178669-11bd-4bc1-a65e-4d92202b8a83","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:03:18.622Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"97178669-11bd-4bc1-a65e-4d92202b8a83"}
{"level":"info","ts":"2026-11-03T15:03:18.622Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"97178669-11bd-4bc1-a65e-4d92202b8a83","RayCluster":"counter-5gq9h","State":""}
{"level":"info","ts":"2026-11-03T15:03:18.622Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"97178669-11bd-4bc1-a65e-4d92202b8a83","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:03:21.623Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"fbf52246-ee26-4bd8-a013-d78ebf6cbe87","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:03:21.623Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"fbf52246-ee26-4bd8-a013-d78ebf6cbe87","name":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:03:21.623Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"fbf52246-ee26-4bd8-a013-d78ebf6cbe87","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:03:21.623Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"fbf52246-ee26-4bd8-a013-d78ebf6cbe87"}
{"level":"info","ts":"2026-11-03T15:03:21.623Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"fbf52246-ee26-4bd8-a013-d78ebf6cbe87","RayCluster":"counter-5gq9h","State":""}
{"level":"info","ts":"2026-11-03T15:03:21.623Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"fbf52246-ee26-4bd8-a013-d78ebf6cbe87","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:03:24.624Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"7283de0e-40b1-48af-9355-27e7dc080098","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:03:24.624Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"7283de0e-40b1-48af-9355-27e7dc080098","name":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:03:24.624Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"7283de0e-40b1-48af-9355-27e7dc080098","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:03:24.624Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"7283de0e-40b1-48af-9355-27e7dc080098"}
{"level":"info","ts":"2026-11-03T15:03:24.624Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"7283de0e-40b1-48af-9355-27e7dc080098","RayCluster":"counter-5gq9h","State":""}
{"level":"info","ts":"2026-11-03T15:03:24.624Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"7283de0e-40b1-48af-9355-27e7dc080098","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:03:27.625Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"af305bdc-a197-4c98-8adc-6c2cbf91b328","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:03:27.625Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"af305bdc-a197-4c98-8adc-6c2cbf91b328","name":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:03:27.625Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"af305bdc-a197-4c98-8adc-6c2cbf91b328","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:03:27.625Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"af305bdc-a197-4c98-8adc-6c2cbf91b328"}
{"level":"info","ts":"2026-11-03T15:03:27.625Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"af305bdc-a197-4c98-8adc-6c2cbf91b328","RayCluster":"counter-5gq9h","State":""}
{"level":"info","ts":"2026-11-03T15:03:27.625Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"af305bdc-a197-4c98-8adc-6c2cbf91b328","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:03:30.626Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"a6efe63a-48dd-4557-aff8-ea2baeace0f3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:03:30.626Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"a6efe63a-48dd-4557-aff8-ea2baeace0f3","name":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:03:30.626Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"a6efe63a-48dd-4557-aff8-ea2baeace0f3","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:03:30.626Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"a6efe63a-48dd-4557-aff8-ea2baeace0f3"}
{"level":"info","ts":"2026-11-03T15:03:30.626Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"a6efe63a-48dd-4557-aff8-ea2baeace0f3","RayCluster":"counter-5gq9h","State":""}
{"level":"info","ts":"2026-11-03T15:03:30.626Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"a6efe63a-48dd-4557-aff8-ea2baeace0f3","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:03:33.628Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"f352e81a-50ef-4468-a47b-e0704ce53967","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:03:33.628Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"f352e81a-50ef-4468-a47b-e0704ce53967","name":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:03:33.628Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"f352e81a-50ef-4468-a47b-e0704ce53967","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:03:33.628Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"f352e81a-50ef-4468-a47b-e0704ce53967"}
{"level":"info","ts":"2026-11-03T15:03:33.628Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"f352e81a-50ef-4468-a47b-e0704ce53967","RayCluster":"counter-5gq9h","State":""}
{"level":"info","ts":"2026-11-03T15:03:33.628Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"f352e81a-50ef-4468-a47b-e0704ce53967","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:03:36.629Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"dc9ea411-c538-4274-aa68-33d8e44303d9","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:03:36.629Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"dc9ea411-c538-4274-aa68-33d8e44303d9","name":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:03:36.629Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"dc9ea411-c538-4274-aa68-33d8e44303d9","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:03:36.629Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"dc9ea411-c538-4274-aa68-33d8e44303d9"}
{"level":"info","ts":"2026-11-03T15:03:36.629Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"dc9ea411-c538-4274-aa68-33d8e44303d9","RayCluster":"counter-5gq9h","State":""}
{"level":"info","ts":"2026-11-03T15:03:36.629Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"dc9ea411-c538-4274-aa68-33d8e44303d9","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:03:39.630Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"8eb504ac-c2cc-49b1-a5d3-c9da6b6ed27d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:03:39.630Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"8eb504ac-c2cc-49b1-a5d3-c9da6b6ed27d","name":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:03:39.630Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"8eb504ac-c2cc-49b1-a5d3-c9da6b6ed27d","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:03:39.631Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"8eb504ac-c2cc-49b1-a5d3-c9da6b6ed27d"}
{"level":"info","ts":"2026-11-03T15:03:39.631Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"8eb504ac-c2cc-49b1-a5d3-c9da6b6ed27d","RayCluster":"counter-5gq9h","State":""}
{"level":"info","ts":"2026-11-03T15:03:39.631Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"8eb504ac-c2cc-49b1-a5d3-c9da6b6ed27d","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:03:42.631Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"b5ec72c1-3f87-49f0-a813-6c5f2727a407","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:03:42.632Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"b5ec72c1-3f87-49f0-a813-6c5f2727a407","name":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:03:42.632Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"b5ec72c1-3f87-49f0-a813-6c5f2727a407","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:03:42.632Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"b5ec72c1-3f87-49f0-a813-6c5f2727a407"}
{"level":"info","ts":"2026-11-03T15:03:42.632Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"b5ec72c1-3f87-49f0-a813-6c5f2727a407","RayCluster":"counter-5gq9h","State":""}
{"level":"info","ts":"2026-11-03T15:03:42.632Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"b5ec72c1-3f87-49f0-a813-6c5f2727a407","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:03:42.874Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"44976392-8287-4fbb-b828-c0d804744e5d","cluster":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:03:45.633Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"a44326cf-06b7-4c3e-908c-1f7793cb3c67","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:03:45.633Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"a44326cf-06b7-4c3e-908c-1f7793cb3c67","name":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:03:45.633Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"a44326cf-06b7-4c3e-908c-1f7793cb3c67","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:03:45.633Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"a44326cf-06b7-4c3e-908c-1f7793cb3c67"}
{"level":"info","ts":"2026-11-03T15:03:45.633Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"a44326cf-06b7-4c3e-908c-1f7793cb3c67","RayCluster":"counter-5gq9h","State":""}
{"level":"info","ts":"2026-11-03T15:03:45.633Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"a44326cf-06b7-4c3e-908c-1f7793cb3c67","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:03:48.633Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"7c3ebd98-e461-4fce-a5ca-c456add3bef6","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:03:48.634Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"7c3ebd98-e461-4fce-a5ca-c456add3bef6","name":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:03:48.634Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"7c3ebd98-e461-4fce-a5ca-c456add3bef6","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:03:48.634Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"7c3ebd98-e461-4fce-a5ca-c456add3bef6"}
{"level":"info","ts":"2026-11-03T15:03:48.634Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"7c3ebd98-e461-4fce-a5ca-c456add3bef6","RayCluster":"counter-5gq9h","State":""}
{"level":"info","ts":"2026-11-03T15:03:48.634Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"7c3ebd98-e461-4fce-a5ca-c456add3bef6","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:03:51.635Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"464c3abb-0da2-470e-a0cd-0ab7232272b9","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:03:51.635Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"464c3abb-0da2-470e-a0cd-0ab7232272b9","name":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:03:51.635Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"464c3abb-0da2-470e-a0cd-0ab7232272b9","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:03:51.635Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"464c3abb-0da2-470e-a0cd-0ab7232272b9"}
{"level":"info","ts":"2026-11-03T15:03:51.635Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"464c3abb-0da2-470e-a0cd-0ab7232272b9","RayCluster":"counter-5gq9h","State":""}
{"level":"info","ts":"2026-11-03T15:03:51.635Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"464c3abb-0da2-470e-a0cd-0ab7232272b9","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:03:54.636Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"8f76eb1c-5f3c-4ecf-a96e-78a4f7190f0a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:03:54.636Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"8f76eb1c-5f3c-4ecf-a96e-78a4f7190f0a","name":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:03:54.636Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"8f76eb1c-5f3c-4ecf-a96e-78a4f7190f0a","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:03:54.636Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"8f76eb1c-5f3c-4ecf-a96e-78a4f7190f0a"}
{"level":"info","ts":"2026-11-03T15:03:54.636Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"8f76eb1c-5f3c-4ecf-a96e-78a4f7190f0a","RayCluster":"counter-5gq9h","State":""}
{"level":"info","ts":"2026-11-03T15:03:54.636Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"8f76eb1c-5f3c-4ecf-a96e-78a4f7190f0a","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:03:57.637Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"dfdc4b95-73b4-4d8d-882d-2f4cf7970432","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:03:57.637Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"dfdc4b95-73b4-4d8d-882d-2f4cf7970432","name":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:03:57.637Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"dfdc4b95-73b4-4d8d-882d-2f4cf7970432","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:03:57.637Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"dfdc4b95-73b4-4d8d-882d-2f4cf7970432"}
{"level":"info","ts":"2026-11-03T15:03:57.637Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"dfdc4b95-73b4-4d8d-882d-2f4cf7970432","RayCluster":"counter-5gq9h","State":""}
{"level":"info","ts":"2026-11-03T15:03:57.637Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"dfdc4b95-73b4-4d8d-882d-2f4cf7970432","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:04:00.638Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"a391c09b-dc38-4ffc-8092-b384c68aecf3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:04:00.638Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"a391c09b-dc38-4ffc-8092-b384c68aecf3","name":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:04:00.638Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"a391c09b-dc38-4ffc-8092-b384c68aecf3","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:04:00.638Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"a391c09b-dc38-4ffc-8092-b384c68aecf3"}
{"level":"info","ts":"2026-11-03T15:04:00.638Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"a391c09b-dc38-4ffc-8092-b384c68aecf3","RayCluster":"counter-5gq9h","State":""}
{"level":"info","ts":"2026-11-03T15:04:00.638Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"a391c09b-dc38-4ffc-8092-b384c68aecf3","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:04:03.639Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"2a735774-a457-4ab3-a4d8-90ff926a32a2","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:04:03.639Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"2a735774-a457-4ab3-a4d8-90ff926a32a2","name":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:04:03.639Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"2a735774-a457-4ab3-a4d8-90ff926a32a2","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:04:03.640Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"2a735774-a457-4ab3-a4d8-90ff926a32a2"}
{"level":"info","ts":"2026-11-03T15:04:03.640Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"2a735774-a457-4ab3-a4d8-90ff926a32a2","RayCluster":"counter-5gq9h","State":""}
{"level":"info","ts":"2026-11-03T15:04:03.640Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"2a735774-a457-4ab3-a4d8-90ff926a32a2","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:04:06.640Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"3ee4e7a5-c3ab-476e-9217-3c66fd193459","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:04:06.640Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"3ee4e7a5-c3ab-476e-9217-3c66fd193459","name":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:04:06.640Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"3ee4e7a5-c3ab-476e-9217-3c66fd193459","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:04:06.640Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"3ee4e7a5-c3ab-476e-9217-3c66fd193459"}
{"level":"info","ts":"2026-11-03T15:04:06.640Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"3ee4e7a5-c3ab-476e-9217-3c66fd193459","RayCluster":"counter-5gq9h","State":""}
{"level":"info","ts":"2026-11-03T15:04:06.640Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"3ee4e7a5-c3ab-476e-9217-3c66fd193459","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:04:09.641Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"48ac4fdb-ee19-446c-9271-f88e2849f5ec","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:04:09.641Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"48ac4fdb-ee19-446c-9271-f88e2849f5ec","name":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:04:09.641Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"48ac4fdb-ee19-446c-9271-f88e2849f5ec","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:04:09.642Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"48ac4fdb-ee19-446c-9271-f88e2849f5ec"}
{"level":"info","ts":"2026-11-03T15:04:09.642Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"48ac4fdb-ee19-446c-9271-f88e2849f5ec","RayCluster":"counter-5gq9h","State":""}
{"level":"info","ts":"2026-11-03T15:04:09.642Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"48ac4fdb-ee19-446c-9271-f88e2849f5ec","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:04:12.642Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"60e25528-484f-44bf-acb0-52fbe5532b03","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:04:12.642Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"60e25528-484f-44bf-acb0-52fbe5532b03","name":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:04:12.642Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"60e25528-484f-44bf-acb0-52fbe5532b03","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:04:12.643Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"60e25528-484f-44bf-acb0-52fbe5532b03"}
{"level":"info","ts":"2026-11-03T15:04:12.643Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"60e25528-484f-44bf-acb0-52fbe5532b03","RayCluster":"counter-5gq9h","State":""}
{"level":"info","ts":"2026-11-03T15:04:12.643Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"60e25528-484f-44bf-acb0-52fbe5532b03","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:04:15.644Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"62e991b6-96b5-4715-846c-7a3b492f4b98","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:04:15.644Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"62e991b6-96b5-4715-846c-7a3b492f4b98","name":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:04:15.644Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"62e991b6-96b5-4715-846c-7a3b492f4b98","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:04:15.644Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"62e991b6-96b5-4715-846c-7a3b492f4b98"}
{"level":"info","ts":"2026-11-03T15:04:15.644Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"62e991b6-96b5-4715-846c-7a3b492f4b98","RayCluster":"counter-5gq9h","State":""}
{"level":"info","ts":"2026-11-03T15:04:15.644Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"62e991b6-96b5-4715-846c-7a3b492f4b98","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:04:18.644Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"8206b9a5-f130-474b-9ada-47e243ef062d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:04:18.644Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"8206b9a5-f130-474b-9ada-47e243ef062d","name":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:04:18.644Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"8206b9a5-f130-474b-9ada-47e243ef062d","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:04:18.645Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"8206b9a5-f130-474b-9ada-47e243ef062d"}
{"level":"info","ts":"2026-11-03T15:04:18.645Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"8206b9a5-f130-474b-9ada-47e243ef062d","RayCluster":"counter-5gq9h","State":""}
{"level":"info","ts":"2026-11-03T15:04:18.645Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"8206b9a5-f130-474b-9ada-47e243ef062d","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:04:21.646Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"1aedf7ef-1b5f-423f-8674-1e74c2a92562","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:04:21.646Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"1aedf7ef-1b5f-423f-8674-1e74c2a92562","name":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:04:21.646Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"1aedf7ef-1b5f-423f-8674-1e74c2a92562","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:04:21.646Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"1aedf7ef-1b5f-423f-8674-1e74c2a92562"}
{"level":"info","ts":"2026-11-03T15:04:21.646Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"1aedf7ef-1b5f-423f-8674-1e74c2a92562","RayCluster":"counter-5gq9h","State":""}
{"level":"info","ts":"2026-11-03T15:04:21.646Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"1aedf7ef-1b5f-423f-8674-1e74c2a92562","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:04:24.647Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"0874d573-9207-4af3-bd36-97de7bc7341c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:04:24.647Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"0874d573-9207-4af3-bd36-97de7bc7341c","name":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:04:24.647Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"0874d573-9207-4af3-bd36-97de7bc7341c","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:04:24.647Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"0874d573-9207-4af3-bd36-97de7bc7341c"}
{"level":"info","ts":"2026-11-03T15:04:24.647Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"0874d573-9207-4af3-bd36-97de7bc7341c","RayCluster":"counter-5gq9h","State":""}
{"level":"info","ts":"2026-11-03T15:04:24.647Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"0874d573-9207-4af3-bd36-97de7bc7341c","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:04:27.648Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"5cbc6f00-aaa6-49ba-8900-4ceb459430da","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:04:27.649Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"5cbc6f00-aaa6-49ba-8900-4ceb459430da","name":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:04:27.649Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"5cbc6f00-aaa6-49ba-8900-4ceb459430da","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:04:27.649Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"5cbc6f00-aaa6-49ba-8900-4ceb459430da"}
{"level":"info","ts":"2026-11-03T15:04:27.649Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"5cbc6f00-aaa6-49ba-8900-4ceb459430da","RayCluster":"counter-5gq9h","State":""}
{"level":"info","ts":"2026-11-03T15:04:27.649Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"5cbc6f00-aaa6-49ba-8900-4ceb459430da","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:04:30.649Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"4bff6548-b499-4a25-82ec-81d96e99471a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:04:30.649Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"4bff6548-b499-4a25-82ec-81d96e99471a","name":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:04:30.649Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"4bff6548-b499-4a25-82ec-81d96e99471a","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:04:30.650Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"4bff6548-b499-4a25-82ec-81d96e99471a"}
{"level":"info","ts":"2026-11-03T15:04:30.650Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"4bff6548-b499-4a25-82ec-81d96e99471a","RayCluster":"counter-5gq9h","State":""}
{"level":"info","ts":"2026-11-03T15:04:30.650Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"4bff6548-b499-4a25-82ec-81d96e99471a","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:04:33.023Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"9fad8f28-00e3-45b6-b1f7-7925ef5ebcd0"}
{"level":"info","ts":"2026-11-03T15:04:33.023Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"9fad8f28-00e3-45b6-b1f7-7925ef5ebcd0"}
{"level":"info","ts":"2026-11-03T15:04:33.023Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"9fad8f28-00e3-45b6-b1f7-7925ef5ebcd0","1 head service found":"counter-5gq9h-head-svc"}
{"level":"info","ts":"2026-11-03T15:04:33.023Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"9fad8f28-00e3-45b6-b1f7-7925ef5ebcd0","Found 1 head Pod":"counter-5gq9h-head-9twx4","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:04:33.023Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"9fad8f28-00e3-45b6-b1f7-7925ef5ebcd0","head Pod":"counter-5gq9h-head-9twx4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-5gq9h-head-9twx4. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:04:33.023Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"9fad8f28-00e3-45b6-b1f7-7925ef5ebcd0","podName":"counter-5gq9h-head-9twx4","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:04:33.023Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"9fad8f28-00e3-45b6-b1f7-7925ef5ebcd0","podName":"counter-5gq9h-head-9twx4","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:04:33.023Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"9fad8f28-00e3-45b6-b1f7-7925ef5ebcd0","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:04:33.023Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"9fad8f28-00e3-45b6-b1f7-7925ef5ebcd0","seconds":300}
{"level":"info","ts":"2026-11-03T15:04:33.651Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"985968cf-a490-4e1d-a4e1-b7df91749d4d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:04:33.651Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"985968cf-a490-4e1d-a4e1-b7df91749d4d","name":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:04:33.651Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"985968cf-a490-4e1d-a4e1-b7df91749d4d","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:04:33.651Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"985968cf-a490-4e1d-a4e1-b7df91749d4d"}
{"level":"info","ts":"2026-11-03T15:04:33.651Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"985968cf-a490-4e1d-a4e1-b7df91749d4d","RayCluster":"counter-5gq9h","State":""}
{"level":"info","ts":"2026-11-03T15:04:33.651Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"985968cf-a490-4e1d-a4e1-b7df91749d4d","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:04:36.652Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"2c56c5ff-709f-47a2-852d-84a1d993fb62","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:04:36.652Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"2c56c5ff-709f-47a2-852d-84a1d993fb62","name":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:04:36.652Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"2c56c5ff-709f-47a2-852d-84a1d993fb62","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:04:36.652Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"2c56c5ff-709f-47a2-852d-84a1d993fb62"}
{"level":"info","ts":"2026-11-03T15:04:36.652Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"2c56c5ff-709f-47a2-852d-84a1d993fb62","RayCluster":"counter-5gq9h","State":""}
{"level":"info","ts":"2026-11-03T15:04:36.652Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"2c56c5ff-709f-47a2-852d-84a1d993fb62","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:04:37.101Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"b164050b-55b9-4aec-ba25-82a766c24064"}
{"level":"info","ts":"2026-11-03T15:04:37.101Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"b164050b-55b9-4aec-ba25-82a766c24064"}
{"level":"info","ts":"2026-11-03T15:04:37.101Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"b164050b-55b9-4aec-ba25-82a766c24064","1 head service found":"counter-5gq9h-head-svc"}
{"level":"info","ts":"2026-11-03T15:04:37.102Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"b164050b-55b9-4aec-ba25-82a766c24064","Found 1 head Pod":"counter-5gq9h-head-9twx4","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:04:37.102Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"b164050b-55b9-4aec-ba25-82a766c24064","head Pod":"counter-5gq9h-head-9twx4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-5gq9h-head-9twx4. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:04:37.102Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"b164050b-55b9-4aec-ba25-82a766c24064","podName":"counter-5gq9h-head-9twx4","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:03:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:04:37.102Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"b164050b-55b9-4aec-ba25-82a766c24064","podName":"counter-5gq9h-head-9twx4","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:03:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:04:37.102Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"b164050b-55b9-4aec-ba25-82a766c24064","name":"counter-5gq9h","old status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1},"new status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:04:37Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:04:37.184Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"59100cf2-869d-44c9-b8b7-a010a5c7ff1a","cluster":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:04:37.185Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"9a8c4bd4-02d6-4b83-9ae7-b5ba24039703","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:04:37.185Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"9a8c4bd4-02d6-4b83-9ae7-b5ba24039703","name":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:04:37.185Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"9a8c4bd4-02d6-4b83-9ae7-b5ba24039703","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:04:37.185Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"9a8c4bd4-02d6-4b83-9ae7-b5ba24039703"}
{"level":"info","ts":"2026-11-03T15:04:37.185Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"9a8c4bd4-02d6-4b83-9ae7-b5ba24039703","RayCluster":"counter-5gq9h","State":""}
{"level":"info","ts":"2026-11-03T15:04:37.185Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"9a8c4bd4-02d6-4b83-9ae7-b5ba24039703","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:03:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:04:37Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:04:37.185Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"9a8c4bd4-02d6-4b83-9ae7-b5ba24039703","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:04:37.245Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"1d1c01d0-421e-4147-a1a7-d820b0ef5261","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:04:37.245Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"1d1c01d0-421e-4147-a1a7-d820b0ef5261","name":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:04:37.245Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"1d1c01d0-421e-4147-a1a7-d820b0ef5261","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:04:37.245Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"1d1c01d0-421e-4147-a1a7-d820b0ef5261"}
{"level":"info","ts":"2026-11-03T15:04:37.245Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"1d1c01d0-421e-4147-a1a7-d820b0ef5261","RayCluster":"counter-5gq9h","State":""}
{"level":"info","ts":"2026-11-03T15:04:37.245Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"1d1c01d0-421e-4147-a1a7-d820b0ef5261","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:04:37Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:04:37Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:04:39.185Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"8b26efaf-4bf3-4b7a-9270-11537ceeab57"}
{"level":"info","ts":"2026-11-03T15:04:39.185Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"8b26efaf-4bf3-4b7a-9270-11537ceeab57"}
{"level":"info","ts":"2026-11-03T15:04:39.185Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"8b26efaf-4bf3-4b7a-9270-11537ceeab57","1 head service found":"counter-5gq9h-head-svc"}
{"level":"info","ts":"2026-11-03T15:04:39.186Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"8b26efaf-4bf3-4b7a-9270-11537ceeab57","Found 1 head Pod":"counter-5gq9h-head-9twx4","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:04:39.186Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"8b26efaf-4bf3-4b7a-9270-11537ceeab57","head Pod":"counter-5gq9h-head-9twx4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-5gq9h-head-9twx4. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:04:39.186Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"8b26efaf-4bf3-4b7a-9270-11537ceeab57","podName":"counter-5gq9h-head-9twx4","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:03:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:04:39.186Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"8b26efaf-4bf3-4b7a-9270-11537ceeab57","podName":"counter-5gq9h-head-9twx4","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:03:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:04:39.186Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"8b26efaf-4bf3-4b7a-9270-11537ceeab57","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:04:39.186Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"8b26efaf-4bf3-4b7a-9270-11537ceeab57","seconds":300}
{"level":"info","ts":"2026-11-03T15:04:40.247Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"5afe9ff0-5ec9-474c-9f7c-e70af32d77f6","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:04:40.247Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"5afe9ff0-5ec9-474c-9f7c-e70af32d77f6","name":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:04:40.247Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"5afe9ff0-5ec9-474c-9f7c-e70af32d77f6","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:04:40.247Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"5afe9ff0-5ec9-474c-9f7c-e70af32d77f6"}
{"level":"info","ts":"2026-11-03T15:04:40.247Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"5afe9ff0-5ec9-474c-9f7c-e70af32d77f6","RayCluster":"counter-5gq9h","State":""}
{"level":"info","ts":"2026-11-03T15:04:40.247Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"5afe9ff0-5ec9-474c-9f7c-e70af32d77f6","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:04:37Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:04:37Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:04:43.248Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"188aae36-4b74-47b6-a505-1003d426e9f8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:04:43.248Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"188aae36-4b74-47b6-a505-1003d426e9f8","name":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:04:43.248Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"188aae36-4b74-47b6-a505-1003d426e9f8","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:04:43.248Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"188aae36-4b74-47b6-a505-1003d426e9f8"}
{"level":"info","ts":"2026-11-03T15:04:43.248Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"188aae36-4b74-47b6-a505-1003d426e9f8","RayCluster":"counter-5gq9h","State":""}
{"level":"info","ts":"2026-11-03T15:04:43.248Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"188aae36-4b74-47b6-a505-1003d426e9f8","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:04:37Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:04:37Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:04:46.249Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"2fbd3ebf-a64d-4390-bd75-ffd42eafe63f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:04:46.249Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"2fbd3ebf-a64d-4390-bd75-ffd42eafe63f","name":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:04:46.249Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"2fbd3ebf-a64d-4390-bd75-ffd42eafe63f","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:04:46.249Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"2fbd3ebf-a64d-4390-bd75-ffd42eafe63f"}
{"level":"info","ts":"2026-11-03T15:04:46.249Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"2fbd3ebf-a64d-4390-bd75-ffd42eafe63f","RayCluster":"counter-5gq9h","State":""}
{"level":"info","ts":"2026-11-03T15:04:46.249Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"2fbd3ebf-a64d-4390-bd75-ffd42eafe63f","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:04:37Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:04:37Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:04:48.110Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"8cabc658-68b5-440f-9d7d-230b5cf00b1f"}
{"level":"info","ts":"2026-11-03T15:04:48.110Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"8cabc658-68b5-440f-9d7d-230b5cf00b1f"}
{"level":"info","ts":"2026-11-03T15:04:48.110Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"8cabc658-68b5-440f-9d7d-230b5cf00b1f","1 head service found":"counter-5gq9h-head-svc"}
{"level":"info","ts":"2026-11-03T15:04:48.110Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"8cabc658-68b5-440f-9d7d-230b5cf00b1f","Found 1 head Pod":"counter-5gq9h-head-9twx4","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:04:48.111Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"8cabc658-68b5-440f-9d7d-230b5cf00b1f","head Pod":"counter-5gq9h-head-9twx4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-5gq9h-head-9twx4. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:04:48.111Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"8cabc658-68b5-440f-9d7d-230b5cf00b1f","name":"counter-5gq9h","old status":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:04:37Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:04:48Z","stateTransitionTimes":{"ready":"2026-11-03T15:04:48Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:04:48Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:04:48Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:04:48.171Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"cb05d918-81a0-4d22-b7f9-20291f7e2ad6","cluster":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:04:48.172Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"d28c221b-f372-42f6-9336-1804fb3476fe","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:04:48.172Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"d28c221b-f372-42f6-9336-1804fb3476fe","name":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:04:48.172Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"d28c221b-f372-42f6-9336-1804fb3476fe","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:04:48.172Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"d28c221b-f372-42f6-9336-1804fb3476fe"}
{"level":"info","ts":"2026-11-03T15:04:48.172Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"d28c221b-f372-42f6-9336-1804fb3476fe","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:04:37Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","dashboardURL":"counter-5gq9h-head-svc.test-ns-mvz6h.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:04:37Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:04:48.172Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"d28c221b-f372-42f6-9336-1804fb3476fe","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-11-03T15:04:48.232Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"a6b90613-aa1f-4330-96ee-45390aff3a6b","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:04:48.232Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"a6b90613-aa1f-4330-96ee-45390aff3a6b","name":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:04:48.232Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"a6b90613-aa1f-4330-96ee-45390aff3a6b","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:04:48.232Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"a6b90613-aa1f-4330-96ee-45390aff3a6b"}
{"level":"info","ts":"2026-11-03T15:04:48.233Z","logger":"controllers.RayJob","msg":"Create a Kubernetes Service","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"a6b90613-aa1f-4330-96ee-45390aff3a6b","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:04:48.304Z","logger":"controllers.RayJob","msg":"The Ray job was not found. Submit a Ray job via an HTTP request.","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"a6b90613-aa1f-4330-96ee-45390aff3a6b","JobId":"counter-nfv8j"}
{"level":"info","ts":"2026-11-03T15:04:50.171Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"2799b528-2c05-436c-9600-3d0c1a83cfe8"}
{"level":"info","ts":"2026-11-03T15:04:50.172Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"2799b528-2c05-436c-9600-3d0c1a83cfe8"}
{"level":"info","ts":"2026-11-03T15:04:50.172Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"2799b528-2c05-436c-9600-3d0c1a83cfe8","1 head service found":"counter-5gq9h-head-svc"}
{"level":"info","ts":"2026-11-03T15:04:50.172Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"2799b528-2c05-436c-9600-3d0c1a83cfe8","Found 1 head Pod":"counter-5gq9h-head-9twx4","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:04:50.172Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"2799b528-2c05-436c-9600-3d0c1a83cfe8","head Pod":"counter-5gq9h-head-9twx4","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-5gq9h-head-9twx4. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:04:50.172Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"2799b528-2c05-436c-9600-3d0c1a83cfe8","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:04:50.172Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"2799b528-2c05-436c-9600-3d0c1a83cfe8","seconds":300}
{"level":"error","ts":"2026-11-03T15:04:50.304Z","logger":"controllers.RayJob","msg":"Failed to submit the Ray job","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"a6b90613-aa1f-4330-96ee-45390aff3a6b","JobId":"counter-nfv8j","error":"Post \"http://counter-5gq9h-head-svc.test-ns-mvz6h.svc.cluster.local:8265/api/jobs/\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.reconcileHTTPModeJobNotFound\n\t/workspace/controllers/ray/rayjob_controller.go:1365\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:306\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-03T15:04:50.304Z","logger":"controllers.RayJob","msg":"Job status check began failing.","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"a6b90613-aa1f-4330-96ee-45390aff3a6b","JobId":"counter-nfv8j","error":"Post \"http://counter-5gq9h-head-svc.test-ns-mvz6h.svc.cluster.local:8265/api/jobs/\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.recordJobStatusCheckFailure\n\t/workspace/controllers/ray/rayjob_controller.go:1381\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.reconcileHTTPModeJobNotFound\n\t/workspace/controllers/ray/rayjob_controller.go:1366\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:306\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:04:50.304Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"a6b90613-aa1f-4330-96ee-45390aff3a6b","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","dashboardURL":"counter-5gq9h-head-svc.test-ns-mvz6h.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:04:37Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","dashboardURL":"counter-5gq9h-head-svc.test-ns-mvz6h.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:04:37Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1},"jobStatusCheckFailureStartTime":"2026-11-03T15:04:50Z"}}
{"level":"info","ts":"2026-11-03T15:04:50.305Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"a6b90613-aa1f-4330-96ee-45390aff3a6b","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-11-03T15:04:50.427Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"438f6107-8159-4c52-a28d-fd2134c6e517","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:04:50.427Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"438f6107-8159-4c52-a28d-fd2134c6e517","name":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:04:50.427Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"438f6107-8159-4c52-a28d-fd2134c6e517","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:04:50.427Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"438f6107-8159-4c52-a28d-fd2134c6e517"}
{"level":"info","ts":"2026-11-03T15:04:50.427Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"438f6107-8159-4c52-a28d-fd2134c6e517","rayCluster":"counter-5gq9h","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:04:50.477Z","logger":"controllers.RayJob","msg":"The Ray job was not found. Submit a Ray job via an HTTP request.","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"438f6107-8159-4c52-a28d-fd2134c6e517","JobId":"counter-nfv8j"}
{"level":"error","ts":"2026-11-03T15:04:51.504Z","logger":"controllers.RayJob","msg":"Failed to submit the Ray job","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"438f6107-8159-4c52-a28d-fd2134c6e517","JobId":"counter-nfv8j","error":"submission ID 'counter-nfv8j' already used, Please use a different submission_id","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.reconcileHTTPModeJobNotFound\n\t/workspace/controllers/ray/rayjob_controller.go:1365\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:306\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-03T15:04:51.504Z","logger":"controllers.RayJob","msg":"Job status check began failing.","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"438f6107-8159-4c52-a28d-fd2134c6e517","JobId":"counter-nfv8j","error":"submission ID 'counter-nfv8j' already used, Please use a different submission_id","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.recordJobStatusCheckFailure\n\t/workspace/controllers/ray/rayjob_controller.go:1381\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.reconcileHTTPModeJobNotFound\n\t/workspace/controllers/ray/rayjob_controller.go:1366\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:306\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:04:51.504Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"438f6107-8159-4c52-a28d-fd2134c6e517","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","dashboardURL":"counter-5gq9h-head-svc.test-ns-mvz6h.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:04:37Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","dashboardURL":"counter-5gq9h-head-svc.test-ns-mvz6h.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:04:37Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1},"jobStatusCheckFailureStartTime":"2026-11-03T15:04:51Z"}}
{"level":"info","ts":"2026-11-03T15:04:51.504Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"438f6107-8159-4c52-a28d-fd2134c6e517","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-11-03T15:04:51.559Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"438f6107-8159-4c52-a28d-fd2134c6e517","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-11-03T15:04:51.559Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"438f6107-8159-4c52-a28d-fd2134c6e517"}
{"level":"error","ts":"2026-11-03T15:04:51.559Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"438f6107-8159-4c52-a28d-fd2134c6e517","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:04:51.560Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"828f71da-5a42-476a-9d12-a3a1b479cdfe","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:04:51.560Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"828f71da-5a42-476a-9d12-a3a1b479cdfe","name":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:04:51.560Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"828f71da-5a42-476a-9d12-a3a1b479cdfe","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:04:51.560Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"828f71da-5a42-476a-9d12-a3a1b479cdfe"}
{"level":"info","ts":"2026-11-03T15:04:51.560Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"828f71da-5a42-476a-9d12-a3a1b479cdfe","rayCluster":"counter-5gq9h","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:04:51.563Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"828f71da-5a42-476a-9d12-a3a1b479cdfe","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","dashboardURL":"counter-5gq9h-head-svc.test-ns-mvz6h.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:04:37Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1},"jobStatusCheckFailureStartTime":"2026-11-03T15:04:50Z"},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:04:51Z"},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","dashboardURL":"counter-5gq9h-head-svc.test-ns-mvz6h.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for resources (CPUs, GPUs, memory, custom resources) to become available. It may be waiting for the runtime environment to be set up.","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:04:48Z","stateTransitionTimes":{"ready":"2026-11-03T15:04:48Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:04:48Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:04:48Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:04:51.563Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"828f71da-5a42-476a-9d12-a3a1b479cdfe","old JobStatus":"","new JobStatus":"PENDING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-11-03T15:04:51.660Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"a0d0695d-9a1b-4086-bc22-210ca4fa45b5","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:04:51.660Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"a0d0695d-9a1b-4086-bc22-210ca4fa45b5","name":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:04:51.660Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"a0d0695d-9a1b-4086-bc22-210ca4fa45b5","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:04:51.660Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"a0d0695d-9a1b-4086-bc22-210ca4fa45b5"}
{"level":"info","ts":"2026-11-03T15:04:51.660Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"a0d0695d-9a1b-4086-bc22-210ca4fa45b5","rayCluster":"counter-5gq9h","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:04:51.663Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"a0d0695d-9a1b-4086-bc22-210ca4fa45b5","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:04:51Z"},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","dashboardURL":"counter-5gq9h-head-svc.test-ns-mvz6h.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for resources (CPUs, GPUs, memory, custom resources) to become available. It may be waiting for the runtime environment to be set up.","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:04:48Z","stateTransitionTimes":{"ready":"2026-11-03T15:04:48Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:04:48Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:04:48Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:04:51Z"},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","dashboardURL":"counter-5gq9h-head-svc.test-ns-mvz6h.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for resources (CPUs, GPUs, memory, custom resources) to become available. It may be waiting for the runtime environment to be set up.","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:04:48Z","stateTransitionTimes":{"ready":"2026-11-03T15:04:48Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:04:48Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:04:48Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:04:54.664Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"8fcb7911-4c68-42ab-9951-b5868686905a","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:04:54.664Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"8fcb7911-4c68-42ab-9951-b5868686905a","name":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:04:54.664Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"8fcb7911-4c68-42ab-9951-b5868686905a","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:04:54.664Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"8fcb7911-4c68-42ab-9951-b5868686905a"}
{"level":"info","ts":"2026-11-03T15:04:54.664Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"8fcb7911-4c68-42ab-9951-b5868686905a","rayCluster":"counter-5gq9h","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:04:54.668Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"8fcb7911-4c68-42ab-9951-b5868686905a","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:04:51Z"},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","dashboardURL":"counter-5gq9h-head-svc.test-ns-mvz6h.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for resources (CPUs, GPUs, memory, custom resources) to become available. It may be waiting for the runtime environment to be set up.","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:04:48Z","stateTransitionTimes":{"ready":"2026-11-03T15:04:48Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:04:48Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:04:48Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:04:51Z"},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","dashboardURL":"counter-5gq9h-head-svc.test-ns-mvz6h.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:04:48Z","stateTransitionTimes":{"ready":"2026-11-03T15:04:48Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:04:48Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:04:48Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:04:54.668Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"8fcb7911-4c68-42ab-9951-b5868686905a","old JobStatus":"PENDING","new JobStatus":"RUNNING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-11-03T15:04:54.764Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"207f72a7-b7a3-4d67-8ec3-a4f3f59b82a3","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:04:54.764Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"207f72a7-b7a3-4d67-8ec3-a4f3f59b82a3","name":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:04:54.765Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"207f72a7-b7a3-4d67-8ec3-a4f3f59b82a3","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:04:54.765Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"207f72a7-b7a3-4d67-8ec3-a4f3f59b82a3"}
{"level":"info","ts":"2026-11-03T15:04:54.765Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"207f72a7-b7a3-4d67-8ec3-a4f3f59b82a3","rayCluster":"counter-5gq9h","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:04:54.768Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"207f72a7-b7a3-4d67-8ec3-a4f3f59b82a3","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:04:51Z"},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","dashboardURL":"counter-5gq9h-head-svc.test-ns-mvz6h.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:04:48Z","stateTransitionTimes":{"ready":"2026-11-03T15:04:48Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:04:48Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:04:48Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:04:51Z"},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","dashboardURL":"counter-5gq9h-head-svc.test-ns-mvz6h.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:04:48Z","stateTransitionTimes":{"ready":"2026-11-03T15:04:48Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:04:48Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:04:48Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:04:57.768Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"db303688-13d1-482c-b2ef-48aa0229c030","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:04:57.768Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"db303688-13d1-482c-b2ef-48aa0229c030","name":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:04:57.768Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"db303688-13d1-482c-b2ef-48aa0229c030","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:04:57.768Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"db303688-13d1-482c-b2ef-48aa0229c030"}
{"level":"info","ts":"2026-11-03T15:04:57.768Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"db303688-13d1-482c-b2ef-48aa0229c030","rayCluster":"counter-5gq9h","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:04:57.771Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"db303688-13d1-482c-b2ef-48aa0229c030","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:04:51Z"},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","dashboardURL":"counter-5gq9h-head-svc.test-ns-mvz6h.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:03:06Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:04:48Z","stateTransitionTimes":{"ready":"2026-11-03T15:04:48Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:04:48Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:04:48Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:04:51Z","endTime":"2026-11-03T15:04:55Z"},"jobId":"counter-nfv8j","rayClusterName":"counter-5gq9h","dashboardURL":"counter-5gq9h-head-svc.test-ns-mvz6h.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Complete","message":"Job finished successfully.","startTime":"2026-11-03T15:03:06Z","succeeded":1,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"500m","desiredMemory":"6G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:04:48Z","stateTransitionTimes":{"ready":"2026-11-03T15:04:48Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.27","serviceIP":"10.134.0.27","podName":"counter-5gq9h-head-9twx4","serviceName":"counter-5gq9h-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:04:48Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:04:48Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:03:07Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:03:07Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:04:57.771Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"db303688-13d1-482c-b2ef-48aa0229c030","old JobStatus":"RUNNING","new JobStatus":"SUCCEEDED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Complete"}
{"level":"info","ts":"2026-11-03T15:04:57.868Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"f9357dda-8bb8-4727-aa55-255648255c72","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:04:57.868Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"f9357dda-8bb8-4727-aa55-255648255c72","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Complete","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-11-03 15:04:57 +0000 UTC","Now":"2026-11-03T15:04:57.868Z","ShutdownTime":"2026-11-03T15:04:57.000Z"}
{"level":"info","ts":"2026-11-03T15:04:57.926Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob is deleted","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"f9357dda-8bb8-4727-aa55-255648255c72","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:04:57.926Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"f9357dda-8bb8-4727-aa55-255648255c72","isClusterDeleted":false}
{"level":"info","ts":"2026-11-03T15:04:57.926Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"f9357dda-8bb8-4727-aa55-255648255c72","deletionMechanism":"ShutdownAfterJobFinishes","RayCluster":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:04:57.927Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"reconcileID":"65816f2c-ff72-445e-9d85-a33219dbc23c"}
{"level":"info","ts":"2026-11-03T15:04:57.927Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-5gq9h","reconcileID":"134a9c51-d4da-4909-9c38-edbd85553d1c","namespacedName":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:04:57.927Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-5gq9h","reconcileID":"134a9c51-d4da-4909-9c38-edbd85553d1c","cluster":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:04:57.927Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-5gq9h","reconcileID":"134a9c51-d4da-4909-9c38-edbd85553d1c","cluster":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:04:57.927Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-5gq9h","reconcileID":"134a9c51-d4da-4909-9c38-edbd85553d1c","namespace":"test-ns-mvz6h","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-11-03T15:04:57.927Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"346866db-1dd6-43e8-9a2b-d7e06aa6ddb2","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:04:57.927Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"346866db-1dd6-43e8-9a2b-d7e06aa6ddb2","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Complete","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-11-03 15:04:57 +0000 UTC","Now":"2026-11-03T15:04:57.927Z","ShutdownTime":"2026-11-03T15:04:57.000Z"}
{"level":"info","ts":"2026-11-03T15:04:57.927Z","logger":"controllers.RayJob","msg":"The deletion of the associated RayCluster for RayJob is ongoing.","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"346866db-1dd6-43e8-9a2b-d7e06aa6ddb2","RayCluster":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:04:57.927Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"346866db-1dd6-43e8-9a2b-d7e06aa6ddb2","isClusterDeleted":false}
{"level":"info","ts":"2026-11-03T15:04:57.927Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"346866db-1dd6-43e8-9a2b-d7e06aa6ddb2","deletionMechanism":"ShutdownAfterJobFinishes","RayCluster":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:04:57.979Z","logger":"authentication-controller","msg":"Deleted shared ReferenceGrant (last cluster with auth)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-5gq9h","reconcileID":"134a9c51-d4da-4909-9c38-edbd85553d1c","name":"kuberay-gateway-access","namespace":"test-ns-mvz6h"}
{"level":"info","ts":"2026-11-03T15:04:58.031Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-5gq9h","reconcileID":"134a9c51-d4da-4909-9c38-edbd85553d1c","name":"test-ns-mvz6h-counter-5gq9h","namespace":"opendatahub"}
{"level":"info","ts":"2026-11-03T15:04:58.083Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-5gq9h","reconcileID":"134a9c51-d4da-4909-9c38-edbd85553d1c","configMap":"kube-rbac-proxy-config-counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:04:58.136Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-5gq9h","reconcileID":"134a9c51-d4da-4909-9c38-edbd85553d1c","serviceAccount":"counter-5gq9h-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-11-03T15:04:58.317Z","logger":"raycluster-resource","msg":"default","name":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:04:58.317Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"counter-5gq9h","namespace":"test-ns-mvz6h"}
{"level":"info","ts":"2026-11-03T15:04:58.482Z","logger":"raycluster-resource","msg":"validate update","name":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:04:58.536Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-5gq9h","reconcileID":"134a9c51-d4da-4909-9c38-edbd85553d1c","cluster":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:04:58.536Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-5gq9h","reconcileID":"5b635a6b-9c2f-4b0f-b9be-39827cb7e7da","namespacedName":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:04:58.536Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-5gq9h","reconcileID":"5b635a6b-9c2f-4b0f-b9be-39827cb7e7da","cluster":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:04:58.536Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-5gq9h","reconcileID":"5b635a6b-9c2f-4b0f-b9be-39827cb7e7da","cluster":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:04:58.536Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-5gq9h","reconcileID":"5b635a6b-9c2f-4b0f-b9be-39827cb7e7da","namespace":"test-ns-mvz6h","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-11-03T15:04:58.536Z","logger":"authentication-controller","msg":"ReferenceGrant already deleted","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-5gq9h","reconcileID":"5b635a6b-9c2f-4b0f-b9be-39827cb7e7da","name":"kuberay-gateway-access"}
{"level":"info","ts":"2026-11-03T15:04:58.537Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"c20144f3-2a50-41f3-ac5a-2224490bc1e5","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:04:58.537Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"c20144f3-2a50-41f3-ac5a-2224490bc1e5","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Complete","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-11-03 15:04:57 +0000 UTC","Now":"2026-11-03T15:04:58.537Z","ShutdownTime":"2026-11-03T15:04:57.000Z"}
{"level":"info","ts":"2026-11-03T15:04:58.537Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"c20144f3-2a50-41f3-ac5a-2224490bc1e5","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:04:58.537Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"c20144f3-2a50-41f3-ac5a-2224490bc1e5","isClusterDeleted":true}
{"level":"info","ts":"2026-11-03T15:04:58.537Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"c20144f3-2a50-41f3-ac5a-2224490bc1e5","deletionMechanism":"ShutdownAfterJobFinishes","RayCluster":"counter-5gq9h"}
{"level":"error","ts":"2026-11-03T15:04:58.595Z","logger":"authentication-controller","msg":"Failed to remove finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-5gq9h","reconcileID":"5b635a6b-9c2f-4b0f-b9be-39827cb7e7da","error":"rayclusters.ray.io \"counter-5gq9h\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleDeletion\n\t/workspace/controllers/ray/authentication_controller.go:195\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:04:58.595Z","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-5gq9h","reconcileID":"5b635a6b-9c2f-4b0f-b9be-39827cb7e7da"}
{"level":"error","ts":"2026-11-03T15:04:58.595Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-5gq9h","reconcileID":"5b635a6b-9c2f-4b0f-b9be-39827cb7e7da","error":"rayclusters.ray.io \"counter-5gq9h\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:04:58.595Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-5gq9h","reconcileID":"3a8957eb-8497-42ba-b7b7-b3cbe510b454","namespacedName":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:04:58.595Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-5gq9h","reconcileID":"3a8957eb-8497-42ba-b7b7-b3cbe510b454"}
{"level":"info","ts":"2026-11-03T15:04:58.595Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-5gq9h","reconcileID":"3a8957eb-8497-42ba-b7b7-b3cbe510b454","namespace":"test-ns-mvz6h","authClusters":0}
{"level":"info","ts":"2026-11-03T15:04:58.595Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-5gq9h","reconcileID":"3a8957eb-8497-42ba-b7b7-b3cbe510b454","namespace":"test-ns-mvz6h"}
{"level":"info","ts":"2026-11-03T15:04:58.834Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"9a15aa7d-3df6-41b3-94c4-a446674ae835","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:04:58.834Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"9a15aa7d-3df6-41b3-94c4-a446674ae835","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-11-03T15:04:58.892Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"9a15aa7d-3df6-41b3-94c4-a446674ae835"}
{"level":"info","ts":"2026-11-03T15:04:58.892Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"9a15aa7d-3df6-41b3-94c4-a446674ae835","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-11-03T15:04:58.892Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"9a15aa7d-3df6-41b3-94c4-a446674ae835","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:04:58.892Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"9a15aa7d-3df6-41b3-94c4-a446674ae835","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:04:58.951Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"587bc413-ee3d-4a11-96ac-e3af4eeb4cfa","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:04:58.951Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"587bc413-ee3d-4a11-96ac-e3af4eeb4cfa","name":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:04:58.951Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"587bc413-ee3d-4a11-96ac-e3af4eeb4cfa","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:04:58.951Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"587bc413-ee3d-4a11-96ac-e3af4eeb4cfa","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:04:59.007Z","logger":"raycluster-resource","msg":"default","name":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:04:59.007Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"fail-g56b8","namespace":"test-ns-mvz6h"}
{"level":"info","ts":"2026-11-03T15:04:59.067Z","logger":"raycluster-resource","msg":"validate create","name":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:04:59.121Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"587bc413-ee3d-4a11-96ac-e3af4eeb4cfa","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:04:59.121Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"4a905142-6909-44ec-9f13-e10e145c92d3"}
{"level":"info","ts":"2026-11-03T15:04:59.121Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"4a905142-6909-44ec-9f13-e10e145c92d3"}
{"level":"info","ts":"2026-11-03T15:04:59.121Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"587bc413-ee3d-4a11-96ac-e3af4eeb4cfa"}
{"level":"info","ts":"2026-11-03T15:04:59.122Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"587bc413-ee3d-4a11-96ac-e3af4eeb4cfa","RayCluster":"fail-g56b8","State":""}
{"level":"info","ts":"2026-11-03T15:04:59.122Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"587bc413-ee3d-4a11-96ac-e3af4eeb4cfa","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:04:59.122Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"fail-g56b8","reconcileID":"16047648-09d5-428b-a3d0-e733a3299203","namespacedName":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:04:59.122Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"fail-g56b8","reconcileID":"16047648-09d5-428b-a3d0-e733a3299203","mode":"IntegratedOAuth","cluster":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:04:59.122Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"fail-g56b8","reconcileID":"16047648-09d5-428b-a3d0-e733a3299203","cluster":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:04:59.122Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"ed43ac56-ff16-4c4c-86e1-b12c269bda32","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:04:59.122Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"ed43ac56-ff16-4c4c-86e1-b12c269bda32","name":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:04:59.122Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"fail-g56b8","reconcileID":"16047648-09d5-428b-a3d0-e733a3299203","cluster":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:04:59.122Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"ed43ac56-ff16-4c4c-86e1-b12c269bda32","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:04:59.122Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"ed43ac56-ff16-4c4c-86e1-b12c269bda32"}
{"level":"info","ts":"2026-11-03T15:04:59.122Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"ed43ac56-ff16-4c4c-86e1-b12c269bda32","RayCluster":"fail-g56b8","State":""}
{"level":"info","ts":"2026-11-03T15:04:59.122Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"ed43ac56-ff16-4c4c-86e1-b12c269bda32","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:04:59.122Z","logger":"controllers.NetworkPolicy","msg":"Reconciling NetworkPolicies for RayCluster","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"7d5cd271-2690-4842-8ea0-4a37bb266a96","cluster":"fail-g56b8","namespace":"test-ns-mvz6h"}
{"level":"info","ts":"2026-11-03T15:04:59.122Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"76e470bd-e9bf-4c8c-bf01-a2505b94c415","cluster":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:04:59.176Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"4a905142-6909-44ec-9f13-e10e145c92d3","name":"fail-g56b8-head-svc"}
{"level":"info","ts":"2026-11-03T15:04:59.176Z","logger":"controllers.NetworkPolicy","msg":"Successfully created NetworkPolicy","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"7d5cd271-2690-4842-8ea0-4a37bb266a96","name":"fail-g56b8-head"}
{"level":"info","ts":"2026-11-03T15:04:59.177Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"4a905142-6909-44ec-9f13-e10e145c92d3"}
{"level":"info","ts":"2026-11-03T15:04:59.180Z","logger":"raycluster-resource","msg":"default","name":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:04:59.180Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"fail-g56b8","namespace":"test-ns-mvz6h"}
{"level":"info","ts":"2026-11-03T15:04:59.228Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"4a905142-6909-44ec-9f13-e10e145c92d3","error":"certificate ray-head-cert-fail-g56b8 not found: certificates.cert-manager.io \"ray-head-cert-fail-g56b8\" not found"}
{"level":"error","ts":"2026-11-03T15:04:59.228Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"4a905142-6909-44ec-9f13-e10e145c92d3","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-fail-g56b8 not found: certificates.cert-manager.io \"ray-head-cert-fail-g56b8\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:04:59.228Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"4a905142-6909-44ec-9f13-e10e145c92d3","error":"mTLS secrets not ready: certificate ray-head-cert-fail-g56b8 not found: certificates.cert-manager.io \"ray-head-cert-fail-g56b8\" not found"}
{"level":"info","ts":"2026-11-03T15:04:59.228Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"4a905142-6909-44ec-9f13-e10e145c92d3","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"fail-g56b8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:04:59.228Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"4a905142-6909-44ec-9f13-e10e145c92d3","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"fail-g56b8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:04:59.228Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"4a905142-6909-44ec-9f13-e10e145c92d3","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"fail-g56b8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:04:59.228Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"4a905142-6909-44ec-9f13-e10e145c92d3","name":"fail-g56b8","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:04:59Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:04:59.232Z","logger":"raycluster-resource","msg":"validate update","name":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:04:59.233Z","logger":"controllers.NetworkPolicy","msg":"Successfully created NetworkPolicy","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"7d5cd271-2690-4842-8ea0-4a37bb266a96","name":"fail-g56b8-workers-small-group"}
{"level":"info","ts":"2026-11-03T15:04:59.233Z","logger":"controllers.NetworkPolicy","msg":"Successfully reconciled NetworkPolicies for RayCluster","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"7d5cd271-2690-4842-8ea0-4a37bb266a96","cluster":"fail-g56b8","namespace":"test-ns-mvz6h"}
{"level":"info","ts":"2026-11-03T15:04:59.233Z","logger":"controllers.NetworkPolicy","msg":"Reconciling NetworkPolicies for RayCluster","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"65cceb4d-7c7a-45b2-a956-9e1bc45a0cce","cluster":"fail-g56b8","namespace":"test-ns-mvz6h"}
{"level":"info","ts":"2026-11-03T15:04:59.286Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"4d9cde74-842f-492a-b771-421a86386170","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:04:59.286Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"4d9cde74-842f-492a-b771-421a86386170","name":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:04:59.286Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"4d9cde74-842f-492a-b771-421a86386170","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:04:59.286Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"4d9cde74-842f-492a-b771-421a86386170"}
{"level":"info","ts":"2026-11-03T15:04:59.286Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"4d9cde74-842f-492a-b771-421a86386170","RayCluster":"fail-g56b8","State":""}
{"level":"info","ts":"2026-11-03T15:04:59.286Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"4d9cde74-842f-492a-b771-421a86386170","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:04:59.290Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"4a905142-6909-44ec-9f13-e10e145c92d3","name":"fail-g56b8","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-g56b8\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-mvz6h","name":"fail-g56b8"}}
{"level":"info","ts":"2026-11-03T15:04:59.290Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"4a905142-6909-44ec-9f13-e10e145c92d3"}
{"level":"error","ts":"2026-11-03T15:04:59.290Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"4a905142-6909-44ec-9f13-e10e145c92d3","error":"mTLS secrets not ready: certificate ray-head-cert-fail-g56b8 not found: certificates.cert-manager.io \"ray-head-cert-fail-g56b8\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:04:59.290Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"288375ec-3a0a-429a-bc9b-160dd2623c03"}
{"level":"info","ts":"2026-11-03T15:04:59.290Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"288375ec-3a0a-429a-bc9b-160dd2623c03"}
{"level":"info","ts":"2026-11-03T15:04:59.290Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"288375ec-3a0a-429a-bc9b-160dd2623c03","1 head service found":"fail-g56b8-head-svc"}
{"level":"info","ts":"2026-11-03T15:04:59.290Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"288375ec-3a0a-429a-bc9b-160dd2623c03"}
{"level":"info","ts":"2026-11-03T15:04:59.338Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"fail-g56b8","reconcileID":"16047648-09d5-428b-a3d0-e733a3299203","name":"fail-g56b8-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-11-03T15:04:59.341Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"288375ec-3a0a-429a-bc9b-160dd2623c03","error":"certificate ray-head-cert-fail-g56b8 not found: certificates.cert-manager.io \"ray-head-cert-fail-g56b8\" not found"}
{"level":"error","ts":"2026-11-03T15:04:59.341Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"288375ec-3a0a-429a-bc9b-160dd2623c03","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-fail-g56b8 not found: certificates.cert-manager.io \"ray-head-cert-fail-g56b8\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:04:59.341Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"288375ec-3a0a-429a-bc9b-160dd2623c03","error":"mTLS secrets not ready: certificate ray-head-cert-fail-g56b8 not found: certificates.cert-manager.io \"ray-head-cert-fail-g56b8\" not found"}
{"level":"info","ts":"2026-11-03T15:04:59.341Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"288375ec-3a0a-429a-bc9b-160dd2623c03","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"fail-g56b8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:04:59.341Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"288375ec-3a0a-429a-bc9b-160dd2623c03","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"fail-g56b8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:04:59.341Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"288375ec-3a0a-429a-bc9b-160dd2623c03","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"fail-g56b8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:04:59.341Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"288375ec-3a0a-429a-bc9b-160dd2623c03","name":"fail-g56b8","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:04:59Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:04:59.342Z","logger":"controllers.NetworkPolicy","msg":"Successfully reconciled NetworkPolicies for RayCluster","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"65cceb4d-7c7a-45b2-a956-9e1bc45a0cce","cluster":"fail-g56b8","namespace":"test-ns-mvz6h"}
{"level":"info","ts":"2026-11-03T15:04:59.342Z","logger":"controllers.NetworkPolicy","msg":"Reconciling NetworkPolicies for RayCluster","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"c6d65ad7-4785-440d-ad27-31d36e272941","cluster":"fail-g56b8","namespace":"test-ns-mvz6h"}
{"level":"info","ts":"2026-11-03T15:04:59.392Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"fail-g56b8","reconcileID":"16047648-09d5-428b-a3d0-e733a3299203","name":"kuberay-gateway-access","namespace":"test-ns-mvz6h","operation":"created"}
{"level":"info","ts":"2026-11-03T15:04:59.402Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"288375ec-3a0a-429a-bc9b-160dd2623c03"}
{"level":"error","ts":"2026-11-03T15:04:59.402Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"288375ec-3a0a-429a-bc9b-160dd2623c03","error":"mTLS secrets not ready: certificate ray-head-cert-fail-g56b8 not found: certificates.cert-manager.io \"ray-head-cert-fail-g56b8\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:04:59.402Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"9aa27208-fa98-4850-8d54-95d3574fe6d3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:04:59.402Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"9aa27208-fa98-4850-8d54-95d3574fe6d3","name":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:04:59.402Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"9aa27208-fa98-4850-8d54-95d3574fe6d3","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:04:59.402Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"9aa27208-fa98-4850-8d54-95d3574fe6d3"}
{"level":"info","ts":"2026-11-03T15:04:59.402Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"9aa27208-fa98-4850-8d54-95d3574fe6d3","RayCluster":"fail-g56b8","State":""}
{"level":"info","ts":"2026-11-03T15:04:59.402Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"9aa27208-fa98-4850-8d54-95d3574fe6d3","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:04:59Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:04:59.402Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"9aa27208-fa98-4850-8d54-95d3574fe6d3","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:04:59.413Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"90ecbcd2-5c82-4d97-9f49-4b6bf931ef9c"}
{"level":"info","ts":"2026-11-03T15:04:59.413Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"90ecbcd2-5c82-4d97-9f49-4b6bf931ef9c"}
{"level":"info","ts":"2026-11-03T15:04:59.413Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"90ecbcd2-5c82-4d97-9f49-4b6bf931ef9c","1 head service found":"fail-g56b8-head-svc"}
{"level":"info","ts":"2026-11-03T15:04:59.413Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"90ecbcd2-5c82-4d97-9f49-4b6bf931ef9c"}
{"level":"info","ts":"2026-11-03T15:04:59.448Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"fail-g56b8","reconcileID":"16047648-09d5-428b-a3d0-e733a3299203","name":"test-ns-mvz6h-fail-g56b8","operation":"created"}
{"level":"info","ts":"2026-11-03T15:04:59.449Z","logger":"controllers.NetworkPolicy","msg":"Successfully reconciled NetworkPolicies for RayCluster","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"c6d65ad7-4785-440d-ad27-31d36e272941","cluster":"fail-g56b8","namespace":"test-ns-mvz6h"}
{"level":"info","ts":"2026-11-03T15:04:59.462Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"4d0d8648-00c0-4bfe-9212-13e2cb908962","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:04:59.462Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"4d0d8648-00c0-4bfe-9212-13e2cb908962","name":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:04:59.462Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"4d0d8648-00c0-4bfe-9212-13e2cb908962","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:04:59.462Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"4d0d8648-00c0-4bfe-9212-13e2cb908962"}
{"level":"info","ts":"2026-11-03T15:04:59.462Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"4d0d8648-00c0-4bfe-9212-13e2cb908962","RayCluster":"fail-g56b8","State":""}
{"level":"info","ts":"2026-11-03T15:04:59.462Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"4d0d8648-00c0-4bfe-9212-13e2cb908962","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:04:59Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:04:59Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:04:59.462Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"90ecbcd2-5c82-4d97-9f49-4b6bf931ef9c","error":"certificate ray-head-cert-fail-g56b8 not found: certificates.cert-manager.io \"ray-head-cert-fail-g56b8\" not found"}
{"level":"error","ts":"2026-11-03T15:04:59.462Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"90ecbcd2-5c82-4d97-9f49-4b6bf931ef9c","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-fail-g56b8 not found: certificates.cert-manager.io \"ray-head-cert-fail-g56b8\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:04:59.462Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"90ecbcd2-5c82-4d97-9f49-4b6bf931ef9c","error":"mTLS secrets not ready: certificate ray-head-cert-fail-g56b8 not found: certificates.cert-manager.io \"ray-head-cert-fail-g56b8\" not found"}
{"level":"info","ts":"2026-11-03T15:04:59.462Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"90ecbcd2-5c82-4d97-9f49-4b6bf931ef9c","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"fail-g56b8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:04:59.462Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"90ecbcd2-5c82-4d97-9f49-4b6bf931ef9c","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"fail-g56b8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:04:59.462Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"90ecbcd2-5c82-4d97-9f49-4b6bf931ef9c","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"fail-g56b8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:04:59.462Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"90ecbcd2-5c82-4d97-9f49-4b6bf931ef9c"}
{"level":"error","ts":"2026-11-03T15:04:59.462Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"90ecbcd2-5c82-4d97-9f49-4b6bf931ef9c","error":"mTLS secrets not ready: certificate ray-head-cert-fail-g56b8 not found: certificates.cert-manager.io \"ray-head-cert-fail-g56b8\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:04:59.483Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"d81d13df-c711-4149-8d77-51014fa4a36c"}
{"level":"info","ts":"2026-11-03T15:04:59.483Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"d81d13df-c711-4149-8d77-51014fa4a36c"}
{"level":"info","ts":"2026-11-03T15:04:59.483Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"d81d13df-c711-4149-8d77-51014fa4a36c","1 head service found":"fail-g56b8-head-svc"}
{"level":"info","ts":"2026-11-03T15:04:59.483Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"d81d13df-c711-4149-8d77-51014fa4a36c"}
{"level":"info","ts":"2026-11-03T15:04:59.502Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"fail-g56b8","reconcileID":"16047648-09d5-428b-a3d0-e733a3299203","name":"kube-rbac-proxy-config-fail-g56b8","operation":"created"}
{"level":"info","ts":"2026-11-03T15:04:59.533Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"d81d13df-c711-4149-8d77-51014fa4a36c","error":"certificate ray-head-cert-fail-g56b8 not found: certificates.cert-manager.io \"ray-head-cert-fail-g56b8\" not found"}
{"level":"error","ts":"2026-11-03T15:04:59.533Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"d81d13df-c711-4149-8d77-51014fa4a36c","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-fail-g56b8 not found: certificates.cert-manager.io \"ray-head-cert-fail-g56b8\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:04:59.533Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"d81d13df-c711-4149-8d77-51014fa4a36c","error":"mTLS secrets not ready: certificate ray-head-cert-fail-g56b8 not found: certificates.cert-manager.io \"ray-head-cert-fail-g56b8\" not found"}
{"level":"info","ts":"2026-11-03T15:04:59.533Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"d81d13df-c711-4149-8d77-51014fa4a36c","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"fail-g56b8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:04:59.533Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"d81d13df-c711-4149-8d77-51014fa4a36c","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"fail-g56b8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:04:59.533Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"d81d13df-c711-4149-8d77-51014fa4a36c","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"fail-g56b8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:04:59.533Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"d81d13df-c711-4149-8d77-51014fa4a36c"}
{"level":"error","ts":"2026-11-03T15:04:59.533Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"d81d13df-c711-4149-8d77-51014fa4a36c","error":"mTLS secrets not ready: certificate ray-head-cert-fail-g56b8 not found: certificates.cert-manager.io \"ray-head-cert-fail-g56b8\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-03T15:04:59.557Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"fail-g56b8","reconcileID":"16047648-09d5-428b-a3d0-e733a3299203","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"fail-g56b8\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:167\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:04:59.557Z","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"fail-g56b8","reconcileID":"16047648-09d5-428b-a3d0-e733a3299203"}
{"level":"error","ts":"2026-11-03T15:04:59.557Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"fail-g56b8","reconcileID":"16047648-09d5-428b-a3d0-e733a3299203","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"fail-g56b8\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:04:59.557Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"fail-g56b8","reconcileID":"850cc7e5-7fd4-4d34-a10e-6bcfb08e1ccf","namespacedName":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:04:59.557Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"fail-g56b8","reconcileID":"850cc7e5-7fd4-4d34-a10e-6bcfb08e1ccf","mode":"IntegratedOAuth","cluster":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:04:59.557Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"fail-g56b8","reconcileID":"850cc7e5-7fd4-4d34-a10e-6bcfb08e1ccf","cluster":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:04:59.574Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"80566270-0f6b-41f5-a8f8-be39195907cf"}
{"level":"info","ts":"2026-11-03T15:04:59.574Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"80566270-0f6b-41f5-a8f8-be39195907cf"}
{"level":"info","ts":"2026-11-03T15:04:59.574Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"80566270-0f6b-41f5-a8f8-be39195907cf","1 head service found":"fail-g56b8-head-svc"}
{"level":"info","ts":"2026-11-03T15:04:59.574Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"80566270-0f6b-41f5-a8f8-be39195907cf"}
{"level":"info","ts":"2026-11-03T15:04:59.617Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"fail-g56b8","reconcileID":"850cc7e5-7fd4-4d34-a10e-6bcfb08e1ccf","cluster":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:04:59.617Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"01ca94cd-d38c-4228-ae4e-361e3bce0e03","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:04:59.617Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"01ca94cd-d38c-4228-ae4e-361e3bce0e03","name":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:04:59.617Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"01ca94cd-d38c-4228-ae4e-361e3bce0e03","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:04:59.618Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"01ca94cd-d38c-4228-ae4e-361e3bce0e03"}
{"level":"info","ts":"2026-11-03T15:04:59.618Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"01ca94cd-d38c-4228-ae4e-361e3bce0e03","RayCluster":"fail-g56b8","State":""}
{"level":"info","ts":"2026-11-03T15:04:59.618Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"01ca94cd-d38c-4228-ae4e-361e3bce0e03","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:04:59Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:04:59Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:04:59.618Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"01ca94cd-d38c-4228-ae4e-361e3bce0e03","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:04:59.626Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"80566270-0f6b-41f5-a8f8-be39195907cf","error":"certificate ray-head-cert-fail-g56b8 not found: certificates.cert-manager.io \"ray-head-cert-fail-g56b8\" not found"}
{"level":"error","ts":"2026-11-03T15:04:59.626Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"80566270-0f6b-41f5-a8f8-be39195907cf","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-fail-g56b8 not found: certificates.cert-manager.io \"ray-head-cert-fail-g56b8\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:04:59.626Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"80566270-0f6b-41f5-a8f8-be39195907cf","error":"mTLS secrets not ready: certificate ray-head-cert-fail-g56b8 not found: certificates.cert-manager.io \"ray-head-cert-fail-g56b8\" not found"}
{"level":"info","ts":"2026-11-03T15:04:59.627Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"80566270-0f6b-41f5-a8f8-be39195907cf","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"fail-g56b8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:04:59.627Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"80566270-0f6b-41f5-a8f8-be39195907cf","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"fail-g56b8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:04:59.627Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"80566270-0f6b-41f5-a8f8-be39195907cf","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"fail-g56b8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:04:59.627Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"80566270-0f6b-41f5-a8f8-be39195907cf"}
{"level":"error","ts":"2026-11-03T15:04:59.627Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"80566270-0f6b-41f5-a8f8-be39195907cf","error":"mTLS secrets not ready: certificate ray-head-cert-fail-g56b8 not found: certificates.cert-manager.io \"ray-head-cert-fail-g56b8\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:04:59.678Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"722de1c9-531e-4652-89bd-bbe1657e0b9f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:04:59.679Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"722de1c9-531e-4652-89bd-bbe1657e0b9f","name":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:04:59.679Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"722de1c9-531e-4652-89bd-bbe1657e0b9f","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:04:59.679Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"722de1c9-531e-4652-89bd-bbe1657e0b9f"}
{"level":"info","ts":"2026-11-03T15:04:59.679Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"722de1c9-531e-4652-89bd-bbe1657e0b9f","RayCluster":"fail-g56b8","State":""}
{"level":"info","ts":"2026-11-03T15:04:59.679Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"722de1c9-531e-4652-89bd-bbe1657e0b9f","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:04:59Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:04:59Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:04:59.708Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"f058fef9-34d1-465a-a99e-059aa054891e"}
{"level":"info","ts":"2026-11-03T15:04:59.708Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"f058fef9-34d1-465a-a99e-059aa054891e"}
{"level":"info","ts":"2026-11-03T15:04:59.708Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"f058fef9-34d1-465a-a99e-059aa054891e","1 head service found":"fail-g56b8-head-svc"}
{"level":"info","ts":"2026-11-03T15:04:59.708Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"f058fef9-34d1-465a-a99e-059aa054891e"}
{"level":"info","ts":"2026-11-03T15:04:59.757Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"f058fef9-34d1-465a-a99e-059aa054891e","error":"certificate ray-head-cert-fail-g56b8 not found: certificates.cert-manager.io \"ray-head-cert-fail-g56b8\" not found"}
{"level":"error","ts":"2026-11-03T15:04:59.757Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"f058fef9-34d1-465a-a99e-059aa054891e","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-fail-g56b8 not found: certificates.cert-manager.io \"ray-head-cert-fail-g56b8\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:04:59.757Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"f058fef9-34d1-465a-a99e-059aa054891e","error":"mTLS secrets not ready: certificate ray-head-cert-fail-g56b8 not found: certificates.cert-manager.io \"ray-head-cert-fail-g56b8\" not found"}
{"level":"info","ts":"2026-11-03T15:04:59.757Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"f058fef9-34d1-465a-a99e-059aa054891e","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"fail-g56b8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:04:59.757Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"f058fef9-34d1-465a-a99e-059aa054891e","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"fail-g56b8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:04:59.757Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"f058fef9-34d1-465a-a99e-059aa054891e","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"fail-g56b8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:04:59.757Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"f058fef9-34d1-465a-a99e-059aa054891e"}
{"level":"error","ts":"2026-11-03T15:04:59.757Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"f058fef9-34d1-465a-a99e-059aa054891e","error":"mTLS secrets not ready: certificate ray-head-cert-fail-g56b8 not found: certificates.cert-manager.io \"ray-head-cert-fail-g56b8\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:04:59.919Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"3c8c3a29-cc7d-48cc-ab26-262975ed42e3"}
{"level":"info","ts":"2026-11-03T15:04:59.919Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"3c8c3a29-cc7d-48cc-ab26-262975ed42e3"}
{"level":"info","ts":"2026-11-03T15:04:59.919Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"3c8c3a29-cc7d-48cc-ab26-262975ed42e3","1 head service found":"fail-g56b8-head-svc"}
{"level":"info","ts":"2026-11-03T15:04:59.919Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"3c8c3a29-cc7d-48cc-ab26-262975ed42e3"}
{"level":"info","ts":"2026-11-03T15:04:59.969Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"3c8c3a29-cc7d-48cc-ab26-262975ed42e3","error":"certificate ray-head-cert-fail-g56b8 is not ready at current generation 1"}
{"level":"error","ts":"2026-11-03T15:04:59.969Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"3c8c3a29-cc7d-48cc-ab26-262975ed42e3","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-fail-g56b8 is not ready at current generation 1","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:04:59.969Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"3c8c3a29-cc7d-48cc-ab26-262975ed42e3","error":"mTLS secrets not ready: certificate ray-head-cert-fail-g56b8 is not ready at current generation 1"}
{"level":"info","ts":"2026-11-03T15:04:59.969Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"3c8c3a29-cc7d-48cc-ab26-262975ed42e3","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"fail-g56b8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:04:59.969Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"3c8c3a29-cc7d-48cc-ab26-262975ed42e3","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"fail-g56b8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:04:59.969Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"3c8c3a29-cc7d-48cc-ab26-262975ed42e3","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"fail-g56b8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:04:59.969Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"3c8c3a29-cc7d-48cc-ab26-262975ed42e3"}
{"level":"error","ts":"2026-11-03T15:04:59.969Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"3c8c3a29-cc7d-48cc-ab26-262975ed42e3","error":"mTLS secrets not ready: certificate ray-head-cert-fail-g56b8 is not ready at current generation 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:05:00.109Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"76e470bd-e9bf-4c8c-bf01-a2505b94c415","certificate":"ray-head-cert-fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:05:00.109Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"76e470bd-e9bf-4c8c-bf01-a2505b94c415"}
{"level":"info","ts":"2026-11-03T15:05:00.110Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"ff5693e6-0460-41a8-bb76-2c5b0e5b602e","cluster":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:05:00.289Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"c949d48e-b559-4630-bc81-0f819c6226fa"}
{"level":"info","ts":"2026-11-03T15:05:00.289Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"c949d48e-b559-4630-bc81-0f819c6226fa"}
{"level":"info","ts":"2026-11-03T15:05:00.289Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"c949d48e-b559-4630-bc81-0f819c6226fa","1 head service found":"fail-g56b8-head-svc"}
{"level":"info","ts":"2026-11-03T15:05:00.290Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"c949d48e-b559-4630-bc81-0f819c6226fa"}
{"level":"info","ts":"2026-11-03T15:05:00.340Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"c949d48e-b559-4630-bc81-0f819c6226fa","error":"certificate ray-head-cert-fail-g56b8 is not ready at current generation 1"}
{"level":"error","ts":"2026-11-03T15:05:00.340Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"c949d48e-b559-4630-bc81-0f819c6226fa","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-fail-g56b8 is not ready at current generation 1","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:05:00.340Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"c949d48e-b559-4630-bc81-0f819c6226fa","error":"mTLS secrets not ready: certificate ray-head-cert-fail-g56b8 is not ready at current generation 1"}
{"level":"info","ts":"2026-11-03T15:05:00.340Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"c949d48e-b559-4630-bc81-0f819c6226fa","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"fail-g56b8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:05:00.340Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"c949d48e-b559-4630-bc81-0f819c6226fa","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"fail-g56b8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:05:00.340Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"c949d48e-b559-4630-bc81-0f819c6226fa","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"fail-g56b8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:05:00.340Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"c949d48e-b559-4630-bc81-0f819c6226fa"}
{"level":"error","ts":"2026-11-03T15:05:00.340Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"c949d48e-b559-4630-bc81-0f819c6226fa","error":"mTLS secrets not ready: certificate ray-head-cert-fail-g56b8 is not ready at current generation 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:05:00.563Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"ff5693e6-0460-41a8-bb76-2c5b0e5b602e","certificate":"ray-head-cert-fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:05:00.563Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"ff5693e6-0460-41a8-bb76-2c5b0e5b602e"}
{"level":"info","ts":"2026-11-03T15:05:00.981Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"8b22d602-7fe6-4de5-867f-b7fe940f1562"}
{"level":"info","ts":"2026-11-03T15:05:00.981Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"8b22d602-7fe6-4de5-867f-b7fe940f1562"}
{"level":"info","ts":"2026-11-03T15:05:00.981Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"8b22d602-7fe6-4de5-867f-b7fe940f1562","1 head service found":"fail-g56b8-head-svc"}
{"level":"info","ts":"2026-11-03T15:05:00.981Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"8b22d602-7fe6-4de5-867f-b7fe940f1562"}
{"level":"info","ts":"2026-11-03T15:05:01.032Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"8b22d602-7fe6-4de5-867f-b7fe940f1562","error":"certificate ray-head-cert-fail-g56b8 is not ready at current generation 1"}
{"level":"error","ts":"2026-11-03T15:05:01.032Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"8b22d602-7fe6-4de5-867f-b7fe940f1562","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-fail-g56b8 is not ready at current generation 1","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:05:01.032Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"8b22d602-7fe6-4de5-867f-b7fe940f1562","error":"mTLS secrets not ready: certificate ray-head-cert-fail-g56b8 is not ready at current generation 1"}
{"level":"info","ts":"2026-11-03T15:05:01.032Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"8b22d602-7fe6-4de5-867f-b7fe940f1562","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"fail-g56b8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:05:01.032Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"8b22d602-7fe6-4de5-867f-b7fe940f1562","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"fail-g56b8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:05:01.032Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"8b22d602-7fe6-4de5-867f-b7fe940f1562","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"fail-g56b8","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:05:01.032Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"8b22d602-7fe6-4de5-867f-b7fe940f1562"}
{"level":"error","ts":"2026-11-03T15:05:01.032Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"8b22d602-7fe6-4de5-867f-b7fe940f1562","error":"mTLS secrets not ready: certificate ray-head-cert-fail-g56b8 is not ready at current generation 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:05:02.313Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"fbf745c8-5254-488f-82fc-5f91cd6af8b7"}
{"level":"info","ts":"2026-11-03T15:05:02.313Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"fbf745c8-5254-488f-82fc-5f91cd6af8b7"}
{"level":"info","ts":"2026-11-03T15:05:02.313Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"fbf745c8-5254-488f-82fc-5f91cd6af8b7","1 head service found":"fail-g56b8-head-svc"}
{"level":"info","ts":"2026-11-03T15:05:02.313Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"fbf745c8-5254-488f-82fc-5f91cd6af8b7"}
{"level":"info","ts":"2026-11-03T15:05:02.414Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"fbf745c8-5254-488f-82fc-5f91cd6af8b7","mode":"IntegratedOAuth","cluster":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:05:02.414Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"fbf745c8-5254-488f-82fc-5f91cd6af8b7","cluster":"fail-g56b8","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-11-03T15:05:02.414Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"fbf745c8-5254-488f-82fc-5f91cd6af8b7","cluster":"fail-g56b8","authType":"IntegratedOAuth","serviceAccount":"fail-g56b8-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-11-03T15:05:02.414Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"fbf745c8-5254-488f-82fc-5f91cd6af8b7","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"fail-g56b8","ray.io/group":"headgroup","ray.io/identifier":"fail-g56b8-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-11-03T15:05:02.414Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"fbf745c8-5254-488f-82fc-5f91cd6af8b7","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-11-03T15:05:02.414Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"fbf745c8-5254-488f-82fc-5f91cd6af8b7","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-11-03T15:05:02.414Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"fbf745c8-5254-488f-82fc-5f91cd6af8b7","rayNodeType":"head","generatedCmd":"ulimit -n ${RAY_START_ULIMIT_OPEN_FILES:-65536}; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-11-03T15:05:02.415Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"fbf745c8-5254-488f-82fc-5f91cd6af8b7","enabled":true}
{"level":"info","ts":"2026-11-03T15:05:02.533Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"fbf745c8-5254-488f-82fc-5f91cd6af8b7","name":"fail-g56b8-head-bkqfp"}
{"level":"info","ts":"2026-11-03T15:05:02.533Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"fbf745c8-5254-488f-82fc-5f91cd6af8b7","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:05:02.533Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"fbf745c8-5254-488f-82fc-5f91cd6af8b7","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:05:02.533Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"fbf745c8-5254-488f-82fc-5f91cd6af8b7","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-11-03T15:05:02.534Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"fbf745c8-5254-488f-82fc-5f91cd6af8b7","group":"small-group","indices":{}}
{"level":"info","ts":"2026-11-03T15:05:02.534Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"fbf745c8-5254-488f-82fc-5f91cd6af8b7","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-11-03T15:05:02.634Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"fbf745c8-5254-488f-82fc-5f91cd6af8b7","creating worker for group":"small-group","index":0,"total":1,"replicaIndex":0}
{"level":"info","ts":"2026-11-03T15:05:02.634Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"fbf745c8-5254-488f-82fc-5f91cd6af8b7","nodeType":"worker","rayStartParams":{"address":"fail-g56b8-head-svc.test-ns-mvz6h.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-11-03T15:05:02.634Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"fbf745c8-5254-488f-82fc-5f91cd6af8b7","rayStartCmd":"ray start  --address=fail-g56b8-head-svc.test-ns-mvz6h.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-11-03T15:05:02.634Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"fbf745c8-5254-488f-82fc-5f91cd6af8b7","rayNodeType":"worker","generatedCmd":"ulimit -n ${RAY_START_ULIMIT_OPEN_FILES:-65536}; ray start  --address=fail-g56b8-head-svc.test-ns-mvz6h.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-11-03T15:05:02.634Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"fbf745c8-5254-488f-82fc-5f91cd6af8b7","enabled":true}
{"level":"info","ts":"2026-11-03T15:05:02.680Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"0eaf0056-d930-4d60-9120-2ed8044a32e7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:05:02.680Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"0eaf0056-d930-4d60-9120-2ed8044a32e7","name":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:05:02.680Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"0eaf0056-d930-4d60-9120-2ed8044a32e7","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:05:02.681Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"0eaf0056-d930-4d60-9120-2ed8044a32e7"}
{"level":"info","ts":"2026-11-03T15:05:02.681Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"0eaf0056-d930-4d60-9120-2ed8044a32e7","RayCluster":"fail-g56b8","State":""}
{"level":"info","ts":"2026-11-03T15:05:02.681Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"0eaf0056-d930-4d60-9120-2ed8044a32e7","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:04:59Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:04:59Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:05:02.696Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"fbf745c8-5254-488f-82fc-5f91cd6af8b7","name":"fail-g56b8-small-group-worker-l77rv"}
{"level":"info","ts":"2026-11-03T15:05:02.696Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"fbf745c8-5254-488f-82fc-5f91cd6af8b7","podName":"fail-g56b8-head-bkqfp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:05:02.696Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"fbf745c8-5254-488f-82fc-5f91cd6af8b7","podName":"fail-g56b8-head-bkqfp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:05:02.696Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"fbf745c8-5254-488f-82fc-5f91cd6af8b7","name":"fail-g56b8","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:04:59Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:05:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:05:02.757Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"2d873e3e-dee8-4d70-bf30-fc34e96f4846"}
{"level":"info","ts":"2026-11-03T15:05:02.757Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"2d873e3e-dee8-4d70-bf30-fc34e96f4846"}
{"level":"info","ts":"2026-11-03T15:05:02.757Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"e738f909-dafd-4a7e-88df-60d5d66bfa6c","cluster":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:05:02.757Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"2d873e3e-dee8-4d70-bf30-fc34e96f4846","1 head service found":"fail-g56b8-head-svc"}
{"level":"info","ts":"2026-11-03T15:05:02.757Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"2d873e3e-dee8-4d70-bf30-fc34e96f4846","Found 1 head Pod":"fail-g56b8-head-bkqfp","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:05:02.757Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"2d873e3e-dee8-4d70-bf30-fc34e96f4846","head Pod":"fail-g56b8-head-bkqfp","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-g56b8-head-bkqfp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:05:02.757Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"2d873e3e-dee8-4d70-bf30-fc34e96f4846","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:05:02.757Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"2d873e3e-dee8-4d70-bf30-fc34e96f4846","worker Pod":"fail-g56b8-small-group-worker-l77rv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-g56b8-small-group-worker-l77rv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:05:02.757Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"2d873e3e-dee8-4d70-bf30-fc34e96f4846","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:05:02.757Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"2d873e3e-dee8-4d70-bf30-fc34e96f4846","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:05:02.757Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"2d873e3e-dee8-4d70-bf30-fc34e96f4846","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:05:02.757Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"2d873e3e-dee8-4d70-bf30-fc34e96f4846","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:05:02.757Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"dedc3ad4-710b-47c8-afc6-f2e02d413c07","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:05:02.757Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"dedc3ad4-710b-47c8-afc6-f2e02d413c07","name":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:05:02.757Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"2d873e3e-dee8-4d70-bf30-fc34e96f4846","podName":"fail-g56b8-head-bkqfp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:05:02.757Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"dedc3ad4-710b-47c8-afc6-f2e02d413c07","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:05:02.757Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"2d873e3e-dee8-4d70-bf30-fc34e96f4846","podName":"fail-g56b8-head-bkqfp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:05:02.758Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"2d873e3e-dee8-4d70-bf30-fc34e96f4846","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:05:02.758Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"2d873e3e-dee8-4d70-bf30-fc34e96f4846","seconds":300}
{"level":"info","ts":"2026-11-03T15:05:02.758Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"dedc3ad4-710b-47c8-afc6-f2e02d413c07"}
{"level":"info","ts":"2026-11-03T15:05:02.758Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"dedc3ad4-710b-47c8-afc6-f2e02d413c07","RayCluster":"fail-g56b8","State":""}
{"level":"info","ts":"2026-11-03T15:05:02.758Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"dedc3ad4-710b-47c8-afc6-f2e02d413c07","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:04:59Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:05:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:05:02.758Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"dedc3ad4-710b-47c8-afc6-f2e02d413c07","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:05:02.769Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"9c0a6465-891c-499e-8c79-c6f00ff1052c"}
{"level":"info","ts":"2026-11-03T15:05:02.769Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"9c0a6465-891c-499e-8c79-c6f00ff1052c"}
{"level":"info","ts":"2026-11-03T15:05:02.769Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"9c0a6465-891c-499e-8c79-c6f00ff1052c","1 head service found":"fail-g56b8-head-svc"}
{"level":"info","ts":"2026-11-03T15:05:02.769Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"9c0a6465-891c-499e-8c79-c6f00ff1052c","Found 1 head Pod":"fail-g56b8-head-bkqfp","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:05:02.769Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"9c0a6465-891c-499e-8c79-c6f00ff1052c","head Pod":"fail-g56b8-head-bkqfp","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-g56b8-head-bkqfp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:05:02.769Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"9c0a6465-891c-499e-8c79-c6f00ff1052c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:05:02.769Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"9c0a6465-891c-499e-8c79-c6f00ff1052c","worker Pod":"fail-g56b8-small-group-worker-l77rv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-g56b8-small-group-worker-l77rv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:05:02.769Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"9c0a6465-891c-499e-8c79-c6f00ff1052c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:05:02.769Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"9c0a6465-891c-499e-8c79-c6f00ff1052c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:05:02.769Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"9c0a6465-891c-499e-8c79-c6f00ff1052c","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:05:02.769Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"9c0a6465-891c-499e-8c79-c6f00ff1052c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:05:02.769Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"9c0a6465-891c-499e-8c79-c6f00ff1052c","podName":"fail-g56b8-head-bkqfp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:05:02.769Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"9c0a6465-891c-499e-8c79-c6f00ff1052c","podName":"fail-g56b8-head-bkqfp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:05:02.769Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"9c0a6465-891c-499e-8c79-c6f00ff1052c","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:05:02.769Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"9c0a6465-891c-499e-8c79-c6f00ff1052c","seconds":300}
{"level":"info","ts":"2026-11-03T15:05:02.817Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"17a0bc81-9e1a-4004-b5e9-2f286821ab7b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:05:02.817Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"17a0bc81-9e1a-4004-b5e9-2f286821ab7b","name":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:05:02.817Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"17a0bc81-9e1a-4004-b5e9-2f286821ab7b","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:05:02.818Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"17a0bc81-9e1a-4004-b5e9-2f286821ab7b"}
{"level":"info","ts":"2026-11-03T15:05:02.818Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"17a0bc81-9e1a-4004-b5e9-2f286821ab7b","RayCluster":"fail-g56b8","State":""}
{"level":"info","ts":"2026-11-03T15:05:02.818Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"17a0bc81-9e1a-4004-b5e9-2f286821ab7b","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:05:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:05:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:05:02.821Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"98dec57d-a93d-4782-8357-fcba74e8eeaf"}
{"level":"info","ts":"2026-11-03T15:05:02.821Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"98dec57d-a93d-4782-8357-fcba74e8eeaf"}
{"level":"info","ts":"2026-11-03T15:05:02.821Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"98dec57d-a93d-4782-8357-fcba74e8eeaf","1 head service found":"fail-g56b8-head-svc"}
{"level":"info","ts":"2026-11-03T15:05:02.821Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"98dec57d-a93d-4782-8357-fcba74e8eeaf","Found 1 head Pod":"fail-g56b8-head-bkqfp","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:05:02.821Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"98dec57d-a93d-4782-8357-fcba74e8eeaf","head Pod":"fail-g56b8-head-bkqfp","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-g56b8-head-bkqfp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:05:02.821Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"98dec57d-a93d-4782-8357-fcba74e8eeaf","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:05:02.821Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"98dec57d-a93d-4782-8357-fcba74e8eeaf","worker Pod":"fail-g56b8-small-group-worker-l77rv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-g56b8-small-group-worker-l77rv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:05:02.821Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"98dec57d-a93d-4782-8357-fcba74e8eeaf","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:05:02.821Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"98dec57d-a93d-4782-8357-fcba74e8eeaf","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:05:02.821Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"98dec57d-a93d-4782-8357-fcba74e8eeaf","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:05:02.821Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"98dec57d-a93d-4782-8357-fcba74e8eeaf","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:05:02.821Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"98dec57d-a93d-4782-8357-fcba74e8eeaf","podName":"fail-g56b8-head-bkqfp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:05:02.821Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"98dec57d-a93d-4782-8357-fcba74e8eeaf","podName":"fail-g56b8-head-bkqfp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:05:02.821Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"98dec57d-a93d-4782-8357-fcba74e8eeaf","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:05:02.821Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"98dec57d-a93d-4782-8357-fcba74e8eeaf","seconds":300}
{"level":"info","ts":"2026-11-03T15:05:03.230Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"d8489567-0f0f-46a4-8561-2bb9d95b1eb3"}
{"level":"info","ts":"2026-11-03T15:05:03.230Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"d8489567-0f0f-46a4-8561-2bb9d95b1eb3"}
{"level":"info","ts":"2026-11-03T15:05:03.230Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"d8489567-0f0f-46a4-8561-2bb9d95b1eb3","1 head service found":"fail-g56b8-head-svc"}
{"level":"info","ts":"2026-11-03T15:05:03.230Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"d8489567-0f0f-46a4-8561-2bb9d95b1eb3","Found 1 head Pod":"fail-g56b8-head-bkqfp","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:05:03.230Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"d8489567-0f0f-46a4-8561-2bb9d95b1eb3","head Pod":"fail-g56b8-head-bkqfp","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-g56b8-head-bkqfp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:05:03.230Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"d8489567-0f0f-46a4-8561-2bb9d95b1eb3","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:05:03.230Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"d8489567-0f0f-46a4-8561-2bb9d95b1eb3","worker Pod":"fail-g56b8-small-group-worker-l77rv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-g56b8-small-group-worker-l77rv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:05:03.230Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"d8489567-0f0f-46a4-8561-2bb9d95b1eb3","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:05:03.230Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"d8489567-0f0f-46a4-8561-2bb9d95b1eb3","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:05:03.230Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"d8489567-0f0f-46a4-8561-2bb9d95b1eb3","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:05:03.230Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"d8489567-0f0f-46a4-8561-2bb9d95b1eb3","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:05:03.230Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"d8489567-0f0f-46a4-8561-2bb9d95b1eb3","podName":"fail-g56b8-head-bkqfp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:05:03.230Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"d8489567-0f0f-46a4-8561-2bb9d95b1eb3","podName":"fail-g56b8-head-bkqfp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:05:03.230Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"d8489567-0f0f-46a4-8561-2bb9d95b1eb3","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:05:03.230Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"d8489567-0f0f-46a4-8561-2bb9d95b1eb3","seconds":300}
{"level":"info","ts":"2026-11-03T15:05:03.445Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"48f6e1e4-03da-47ab-8ce6-ba003aa808b3"}
{"level":"info","ts":"2026-11-03T15:05:03.445Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"48f6e1e4-03da-47ab-8ce6-ba003aa808b3"}
{"level":"info","ts":"2026-11-03T15:05:03.445Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"48f6e1e4-03da-47ab-8ce6-ba003aa808b3","1 head service found":"fail-g56b8-head-svc"}
{"level":"info","ts":"2026-11-03T15:05:03.445Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"48f6e1e4-03da-47ab-8ce6-ba003aa808b3","Found 1 head Pod":"fail-g56b8-head-bkqfp","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:05:03.445Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"48f6e1e4-03da-47ab-8ce6-ba003aa808b3","head Pod":"fail-g56b8-head-bkqfp","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-g56b8-head-bkqfp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:05:03.445Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"48f6e1e4-03da-47ab-8ce6-ba003aa808b3","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:05:03.445Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"48f6e1e4-03da-47ab-8ce6-ba003aa808b3","worker Pod":"fail-g56b8-small-group-worker-l77rv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-g56b8-small-group-worker-l77rv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:05:03.445Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"48f6e1e4-03da-47ab-8ce6-ba003aa808b3","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:05:03.445Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"48f6e1e4-03da-47ab-8ce6-ba003aa808b3","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:05:03.445Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"48f6e1e4-03da-47ab-8ce6-ba003aa808b3","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:05:03.445Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"48f6e1e4-03da-47ab-8ce6-ba003aa808b3","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:05:03.445Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"48f6e1e4-03da-47ab-8ce6-ba003aa808b3","podName":"fail-g56b8-small-group-worker-l77rv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:05:03.445Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"48f6e1e4-03da-47ab-8ce6-ba003aa808b3","podName":"fail-g56b8-small-group-worker-l77rv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:05:03.445Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"48f6e1e4-03da-47ab-8ce6-ba003aa808b3","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:05:03.445Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"48f6e1e4-03da-47ab-8ce6-ba003aa808b3","seconds":300}
{"level":"info","ts":"2026-11-03T15:05:04.128Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"c2f0c9db-85a0-468e-898e-4d20e3fe86f6"}
{"level":"info","ts":"2026-11-03T15:05:04.128Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"c2f0c9db-85a0-468e-898e-4d20e3fe86f6"}
{"level":"info","ts":"2026-11-03T15:05:04.128Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"c2f0c9db-85a0-468e-898e-4d20e3fe86f6","1 head service found":"fail-g56b8-head-svc"}
{"level":"info","ts":"2026-11-03T15:05:04.128Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"c2f0c9db-85a0-468e-898e-4d20e3fe86f6","Found 1 head Pod":"fail-g56b8-head-bkqfp","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:05:04.128Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"c2f0c9db-85a0-468e-898e-4d20e3fe86f6","head Pod":"fail-g56b8-head-bkqfp","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-g56b8-head-bkqfp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:05:04.128Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"c2f0c9db-85a0-468e-898e-4d20e3fe86f6","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:05:04.128Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"c2f0c9db-85a0-468e-898e-4d20e3fe86f6","worker Pod":"fail-g56b8-small-group-worker-l77rv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-g56b8-small-group-worker-l77rv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:05:04.128Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"c2f0c9db-85a0-468e-898e-4d20e3fe86f6","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:05:04.128Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"c2f0c9db-85a0-468e-898e-4d20e3fe86f6","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:05:04.128Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"c2f0c9db-85a0-468e-898e-4d20e3fe86f6","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:05:04.128Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"c2f0c9db-85a0-468e-898e-4d20e3fe86f6","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:05:04.128Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"c2f0c9db-85a0-468e-898e-4d20e3fe86f6","podName":"fail-g56b8-head-bkqfp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:05:04.128Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"c2f0c9db-85a0-468e-898e-4d20e3fe86f6","podName":"fail-g56b8-head-bkqfp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:05:04.128Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"c2f0c9db-85a0-468e-898e-4d20e3fe86f6","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:05:04.128Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"c2f0c9db-85a0-468e-898e-4d20e3fe86f6","seconds":300}
{"level":"info","ts":"2026-11-03T15:05:04.411Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"c0838cb3-e748-40de-8985-5087058894c1"}
{"level":"info","ts":"2026-11-03T15:05:04.411Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"c0838cb3-e748-40de-8985-5087058894c1"}
{"level":"info","ts":"2026-11-03T15:05:04.411Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"c0838cb3-e748-40de-8985-5087058894c1","1 head service found":"fail-g56b8-head-svc"}
{"level":"info","ts":"2026-11-03T15:05:04.412Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"c0838cb3-e748-40de-8985-5087058894c1","Found 1 head Pod":"fail-g56b8-head-bkqfp","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:05:04.412Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"c0838cb3-e748-40de-8985-5087058894c1","head Pod":"fail-g56b8-head-bkqfp","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-g56b8-head-bkqfp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:05:04.412Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"c0838cb3-e748-40de-8985-5087058894c1","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:05:04.412Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"c0838cb3-e748-40de-8985-5087058894c1","worker Pod":"fail-g56b8-small-group-worker-l77rv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-g56b8-small-group-worker-l77rv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:05:04.412Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"c0838cb3-e748-40de-8985-5087058894c1","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:05:04.412Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"c0838cb3-e748-40de-8985-5087058894c1","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:05:04.412Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"c0838cb3-e748-40de-8985-5087058894c1","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:05:04.412Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"c0838cb3-e748-40de-8985-5087058894c1","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:05:04.412Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"c0838cb3-e748-40de-8985-5087058894c1","podName":"fail-g56b8-small-group-worker-l77rv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:05:04.412Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"c0838cb3-e748-40de-8985-5087058894c1","podName":"fail-g56b8-small-group-worker-l77rv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:05:04.412Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"c0838cb3-e748-40de-8985-5087058894c1","name":"fail-g56b8","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:05:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:05:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:05:04.473Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"c652d973-ebc5-45be-952e-e08049ffc04c","cluster":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:05:04.473Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"926fb5ba-e05d-47bc-999c-78b6bf6710ee","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:05:04.473Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"926fb5ba-e05d-47bc-999c-78b6bf6710ee","name":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:05:04.473Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"926fb5ba-e05d-47bc-999c-78b6bf6710ee","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:05:04.473Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"926fb5ba-e05d-47bc-999c-78b6bf6710ee"}
{"level":"info","ts":"2026-11-03T15:05:04.473Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"926fb5ba-e05d-47bc-999c-78b6bf6710ee","RayCluster":"fail-g56b8","State":""}
{"level":"info","ts":"2026-11-03T15:05:04.473Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"926fb5ba-e05d-47bc-999c-78b6bf6710ee","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:05:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:05:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:05:04.473Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"926fb5ba-e05d-47bc-999c-78b6bf6710ee","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:05:04.533Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"953d9424-6ded-4f95-81ff-8bddbcac55ac","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:05:04.533Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"953d9424-6ded-4f95-81ff-8bddbcac55ac","name":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:05:04.533Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"953d9424-6ded-4f95-81ff-8bddbcac55ac","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:05:04.534Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"953d9424-6ded-4f95-81ff-8bddbcac55ac"}
{"level":"info","ts":"2026-11-03T15:05:04.534Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"953d9424-6ded-4f95-81ff-8bddbcac55ac","RayCluster":"fail-g56b8","State":""}
{"level":"info","ts":"2026-11-03T15:05:04.534Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"953d9424-6ded-4f95-81ff-8bddbcac55ac","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:05:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:05:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:05:04.780Z","logger":"controllers.RayClusterMTLS","msg":"Updating head certificate","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"c652d973-ebc5-45be-952e-e08049ffc04c","certificate":"ray-head-cert-fail-g56b8","ipAddresses":["10.132.0.32","127.0.0.1"]}
{"level":"info","ts":"2026-11-03T15:05:05.036Z","logger":"controllers.RayClusterMTLS","msg":"Updating worker certificate","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"c652d973-ebc5-45be-952e-e08049ffc04c","certificate":"ray-worker-cert-fail-g56b8","ipAddresses":["10.134.0.28","127.0.0.1"]}
{"level":"info","ts":"2026-11-03T15:05:05.293Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"c652d973-ebc5-45be-952e-e08049ffc04c","certificate":"ray-head-cert-fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:05:05.293Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"c652d973-ebc5-45be-952e-e08049ffc04c"}
{"level":"info","ts":"2026-11-03T15:05:06.473Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"6f9988b7-eaa2-41e1-89c3-2968f43d88fc"}
{"level":"info","ts":"2026-11-03T15:05:06.473Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"6f9988b7-eaa2-41e1-89c3-2968f43d88fc"}
{"level":"info","ts":"2026-11-03T15:05:06.473Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"6f9988b7-eaa2-41e1-89c3-2968f43d88fc","1 head service found":"fail-g56b8-head-svc"}
{"level":"info","ts":"2026-11-03T15:05:06.473Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"6f9988b7-eaa2-41e1-89c3-2968f43d88fc","Found 1 head Pod":"fail-g56b8-head-bkqfp","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:05:06.473Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"6f9988b7-eaa2-41e1-89c3-2968f43d88fc","head Pod":"fail-g56b8-head-bkqfp","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-g56b8-head-bkqfp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:05:06.473Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"6f9988b7-eaa2-41e1-89c3-2968f43d88fc","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:05:06.473Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"6f9988b7-eaa2-41e1-89c3-2968f43d88fc","worker Pod":"fail-g56b8-small-group-worker-l77rv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-g56b8-small-group-worker-l77rv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:05:06.473Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"6f9988b7-eaa2-41e1-89c3-2968f43d88fc","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:05:06.473Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"6f9988b7-eaa2-41e1-89c3-2968f43d88fc","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:05:06.473Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"6f9988b7-eaa2-41e1-89c3-2968f43d88fc","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:05:06.473Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"6f9988b7-eaa2-41e1-89c3-2968f43d88fc","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:05:06.474Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"6f9988b7-eaa2-41e1-89c3-2968f43d88fc","podName":"fail-g56b8-head-bkqfp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:05:06.474Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"6f9988b7-eaa2-41e1-89c3-2968f43d88fc","podName":"fail-g56b8-head-bkqfp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:05:06.474Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"6f9988b7-eaa2-41e1-89c3-2968f43d88fc","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:05:06.474Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"6f9988b7-eaa2-41e1-89c3-2968f43d88fc","seconds":300}
{"level":"info","ts":"2026-11-03T15:05:07.534Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"ccd6268d-c20d-472d-bd26-96b5fc17905d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:05:07.534Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"ccd6268d-c20d-472d-bd26-96b5fc17905d","name":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:05:07.534Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"ccd6268d-c20d-472d-bd26-96b5fc17905d","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:05:07.535Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"ccd6268d-c20d-472d-bd26-96b5fc17905d"}
{"level":"info","ts":"2026-11-03T15:05:07.535Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"ccd6268d-c20d-472d-bd26-96b5fc17905d","RayCluster":"fail-g56b8","State":""}
{"level":"info","ts":"2026-11-03T15:05:07.535Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"ccd6268d-c20d-472d-bd26-96b5fc17905d","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:05:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:05:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:05:10.536Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"8867a03a-9550-4843-b7d4-05d3908167ed","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:05:10.536Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"8867a03a-9550-4843-b7d4-05d3908167ed","name":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:05:10.536Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"8867a03a-9550-4843-b7d4-05d3908167ed","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:05:10.536Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"8867a03a-9550-4843-b7d4-05d3908167ed"}
{"level":"info","ts":"2026-11-03T15:05:10.536Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"8867a03a-9550-4843-b7d4-05d3908167ed","RayCluster":"fail-g56b8","State":""}
{"level":"info","ts":"2026-11-03T15:05:10.536Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"8867a03a-9550-4843-b7d4-05d3908167ed","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:05:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:05:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:05:13.537Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"f975d06c-3023-4fd5-9124-e8e852ab6ffe","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:05:13.537Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"f975d06c-3023-4fd5-9124-e8e852ab6ffe","name":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:05:13.537Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"f975d06c-3023-4fd5-9124-e8e852ab6ffe","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:05:13.537Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"f975d06c-3023-4fd5-9124-e8e852ab6ffe"}
{"level":"info","ts":"2026-11-03T15:05:13.537Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"f975d06c-3023-4fd5-9124-e8e852ab6ffe","RayCluster":"fail-g56b8","State":""}
{"level":"info","ts":"2026-11-03T15:05:13.537Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"f975d06c-3023-4fd5-9124-e8e852ab6ffe","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:05:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:05:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:05:16.538Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"7e28e670-158f-49d1-a257-9d94cceacfea","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:05:16.538Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"7e28e670-158f-49d1-a257-9d94cceacfea","name":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:05:16.538Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"7e28e670-158f-49d1-a257-9d94cceacfea","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:05:16.538Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"7e28e670-158f-49d1-a257-9d94cceacfea"}
{"level":"info","ts":"2026-11-03T15:05:16.538Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"7e28e670-158f-49d1-a257-9d94cceacfea","RayCluster":"fail-g56b8","State":""}
{"level":"info","ts":"2026-11-03T15:05:16.538Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"7e28e670-158f-49d1-a257-9d94cceacfea","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:05:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:05:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:05:19.539Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"0338fcb4-8328-4278-8ca8-f8192c7fed08","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:05:19.539Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"0338fcb4-8328-4278-8ca8-f8192c7fed08","name":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:05:19.539Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"0338fcb4-8328-4278-8ca8-f8192c7fed08","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:05:19.539Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"0338fcb4-8328-4278-8ca8-f8192c7fed08"}
{"level":"info","ts":"2026-11-03T15:05:19.539Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"0338fcb4-8328-4278-8ca8-f8192c7fed08","RayCluster":"fail-g56b8","State":""}
{"level":"info","ts":"2026-11-03T15:05:19.539Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"0338fcb4-8328-4278-8ca8-f8192c7fed08","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:05:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:05:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:05:22.540Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"4e416089-1dfa-47ce-8844-24d0bbfc59c1","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:05:22.540Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"4e416089-1dfa-47ce-8844-24d0bbfc59c1","name":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:05:22.540Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"4e416089-1dfa-47ce-8844-24d0bbfc59c1","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:05:22.540Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"4e416089-1dfa-47ce-8844-24d0bbfc59c1"}
{"level":"info","ts":"2026-11-03T15:05:22.540Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"4e416089-1dfa-47ce-8844-24d0bbfc59c1","RayCluster":"fail-g56b8","State":""}
{"level":"info","ts":"2026-11-03T15:05:22.540Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"4e416089-1dfa-47ce-8844-24d0bbfc59c1","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:05:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:05:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:05:25.542Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"29eda4fd-4572-4ec7-a472-9cc52f281450","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:05:25.542Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"29eda4fd-4572-4ec7-a472-9cc52f281450","name":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:05:25.542Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"29eda4fd-4572-4ec7-a472-9cc52f281450","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:05:25.542Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"29eda4fd-4572-4ec7-a472-9cc52f281450"}
{"level":"info","ts":"2026-11-03T15:05:25.542Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"29eda4fd-4572-4ec7-a472-9cc52f281450","RayCluster":"fail-g56b8","State":""}
{"level":"info","ts":"2026-11-03T15:05:25.542Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"29eda4fd-4572-4ec7-a472-9cc52f281450","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:05:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:05:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:05:28.543Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"4d3c61b4-21e7-4a12-8474-f253ad90ba27","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:05:28.543Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"4d3c61b4-21e7-4a12-8474-f253ad90ba27","name":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:05:28.543Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"4d3c61b4-21e7-4a12-8474-f253ad90ba27","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:05:28.543Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"4d3c61b4-21e7-4a12-8474-f253ad90ba27"}
{"level":"info","ts":"2026-11-03T15:05:28.543Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"4d3c61b4-21e7-4a12-8474-f253ad90ba27","RayCluster":"fail-g56b8","State":""}
{"level":"info","ts":"2026-11-03T15:05:28.543Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"4d3c61b4-21e7-4a12-8474-f253ad90ba27","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:05:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:05:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:05:31.544Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"212ee150-4502-427e-98c3-28292687d89c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:05:31.544Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"212ee150-4502-427e-98c3-28292687d89c","name":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:05:31.544Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"212ee150-4502-427e-98c3-28292687d89c","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:05:31.544Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"212ee150-4502-427e-98c3-28292687d89c"}
{"level":"info","ts":"2026-11-03T15:05:31.544Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"212ee150-4502-427e-98c3-28292687d89c","RayCluster":"fail-g56b8","State":""}
{"level":"info","ts":"2026-11-03T15:05:31.544Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"212ee150-4502-427e-98c3-28292687d89c","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:05:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:05:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:05:34.545Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"bdd724b1-53e7-4a44-8f2c-2388aa0715e5","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:05:34.545Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"bdd724b1-53e7-4a44-8f2c-2388aa0715e5","name":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:05:34.545Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"bdd724b1-53e7-4a44-8f2c-2388aa0715e5","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:05:34.545Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"bdd724b1-53e7-4a44-8f2c-2388aa0715e5"}
{"level":"info","ts":"2026-11-03T15:05:34.545Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"bdd724b1-53e7-4a44-8f2c-2388aa0715e5","RayCluster":"fail-g56b8","State":""}
{"level":"info","ts":"2026-11-03T15:05:34.545Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"bdd724b1-53e7-4a44-8f2c-2388aa0715e5","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:05:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:05:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:05:35.293Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"6db79442-b639-43b9-8601-b5a8180eb2f1","cluster":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:05:37.546Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"2f020364-cdbf-473f-a778-433afc589c2a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:05:37.546Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"2f020364-cdbf-473f-a778-433afc589c2a","name":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:05:37.546Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"2f020364-cdbf-473f-a778-433afc589c2a","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:05:37.547Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"2f020364-cdbf-473f-a778-433afc589c2a"}
{"level":"info","ts":"2026-11-03T15:05:37.547Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"2f020364-cdbf-473f-a778-433afc589c2a","RayCluster":"fail-g56b8","State":""}
{"level":"info","ts":"2026-11-03T15:05:37.547Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"2f020364-cdbf-473f-a778-433afc589c2a","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:05:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:05:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:05:40.548Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"a9b7c943-2e0d-416b-b4b5-cef0b7a7dee3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:05:40.548Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"a9b7c943-2e0d-416b-b4b5-cef0b7a7dee3","name":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:05:40.548Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"a9b7c943-2e0d-416b-b4b5-cef0b7a7dee3","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:05:40.548Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"a9b7c943-2e0d-416b-b4b5-cef0b7a7dee3"}
{"level":"info","ts":"2026-11-03T15:05:40.548Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"a9b7c943-2e0d-416b-b4b5-cef0b7a7dee3","RayCluster":"fail-g56b8","State":""}
{"level":"info","ts":"2026-11-03T15:05:40.548Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"a9b7c943-2e0d-416b-b4b5-cef0b7a7dee3","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:05:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:05:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:05:43.549Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"2f423113-fc99-4dfd-b14c-947ba13d0963","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:05:43.549Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"2f423113-fc99-4dfd-b14c-947ba13d0963","name":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:05:43.549Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"2f423113-fc99-4dfd-b14c-947ba13d0963","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:05:43.549Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"2f423113-fc99-4dfd-b14c-947ba13d0963"}
{"level":"info","ts":"2026-11-03T15:05:43.549Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"2f423113-fc99-4dfd-b14c-947ba13d0963","RayCluster":"fail-g56b8","State":""}
{"level":"info","ts":"2026-11-03T15:05:43.549Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"2f423113-fc99-4dfd-b14c-947ba13d0963","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:05:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:05:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:05:46.550Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"74ea11a3-f9b3-47a8-85cc-2f70f05a94c1","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:05:46.550Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"74ea11a3-f9b3-47a8-85cc-2f70f05a94c1","name":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:05:46.550Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"74ea11a3-f9b3-47a8-85cc-2f70f05a94c1","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:05:46.550Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"74ea11a3-f9b3-47a8-85cc-2f70f05a94c1"}
{"level":"info","ts":"2026-11-03T15:05:46.550Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"74ea11a3-f9b3-47a8-85cc-2f70f05a94c1","RayCluster":"fail-g56b8","State":""}
{"level":"info","ts":"2026-11-03T15:05:46.550Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"74ea11a3-f9b3-47a8-85cc-2f70f05a94c1","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:05:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:05:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:05:49.551Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"d63c4795-adcc-4d03-ab52-354a99b84542","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:05:49.551Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"d63c4795-adcc-4d03-ab52-354a99b84542","name":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:05:49.551Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"d63c4795-adcc-4d03-ab52-354a99b84542","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:05:49.551Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"d63c4795-adcc-4d03-ab52-354a99b84542"}
{"level":"info","ts":"2026-11-03T15:05:49.551Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"d63c4795-adcc-4d03-ab52-354a99b84542","RayCluster":"fail-g56b8","State":""}
{"level":"info","ts":"2026-11-03T15:05:49.551Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"d63c4795-adcc-4d03-ab52-354a99b84542","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:05:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:05:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:05:52.552Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"0e30dd4d-91c6-4b2e-8d18-3fa39d0236f0","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:05:52.552Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"0e30dd4d-91c6-4b2e-8d18-3fa39d0236f0","name":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:05:52.552Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"0e30dd4d-91c6-4b2e-8d18-3fa39d0236f0","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:05:52.552Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"0e30dd4d-91c6-4b2e-8d18-3fa39d0236f0"}
{"level":"info","ts":"2026-11-03T15:05:52.552Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"0e30dd4d-91c6-4b2e-8d18-3fa39d0236f0","RayCluster":"fail-g56b8","State":""}
{"level":"info","ts":"2026-11-03T15:05:52.552Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"0e30dd4d-91c6-4b2e-8d18-3fa39d0236f0","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:05:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:05:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:05:55.553Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"37b74b81-fa8d-497a-9eba-7b31034cf2e7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:05:55.553Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"37b74b81-fa8d-497a-9eba-7b31034cf2e7","name":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:05:55.553Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"37b74b81-fa8d-497a-9eba-7b31034cf2e7","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:05:55.553Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"37b74b81-fa8d-497a-9eba-7b31034cf2e7"}
{"level":"info","ts":"2026-11-03T15:05:55.553Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"37b74b81-fa8d-497a-9eba-7b31034cf2e7","RayCluster":"fail-g56b8","State":""}
{"level":"info","ts":"2026-11-03T15:05:55.553Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"37b74b81-fa8d-497a-9eba-7b31034cf2e7","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:05:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:05:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:05:58.554Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"d35cf9a4-8528-436f-8e13-f200ad5abf6f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:05:58.554Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"d35cf9a4-8528-436f-8e13-f200ad5abf6f","name":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:05:58.554Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"d35cf9a4-8528-436f-8e13-f200ad5abf6f","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:05:58.555Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"d35cf9a4-8528-436f-8e13-f200ad5abf6f"}
{"level":"info","ts":"2026-11-03T15:05:58.555Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"d35cf9a4-8528-436f-8e13-f200ad5abf6f","RayCluster":"fail-g56b8","State":""}
{"level":"info","ts":"2026-11-03T15:05:58.555Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"d35cf9a4-8528-436f-8e13-f200ad5abf6f","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:05:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:05:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:06:01.555Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"1a950345-f486-4056-9c10-2870265efa68","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:06:01.555Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"1a950345-f486-4056-9c10-2870265efa68","name":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:06:01.555Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"1a950345-f486-4056-9c10-2870265efa68","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:06:01.555Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"1a950345-f486-4056-9c10-2870265efa68"}
{"level":"info","ts":"2026-11-03T15:06:01.555Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"1a950345-f486-4056-9c10-2870265efa68","RayCluster":"fail-g56b8","State":""}
{"level":"info","ts":"2026-11-03T15:06:01.555Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"1a950345-f486-4056-9c10-2870265efa68","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:05:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:05:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:06:04.556Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"09475ebd-cd31-42d2-a5cc-50781ea0fe93","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:06:04.556Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"09475ebd-cd31-42d2-a5cc-50781ea0fe93","name":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:06:04.556Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"09475ebd-cd31-42d2-a5cc-50781ea0fe93","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:06:04.556Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"09475ebd-cd31-42d2-a5cc-50781ea0fe93"}
{"level":"info","ts":"2026-11-03T15:06:04.556Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"09475ebd-cd31-42d2-a5cc-50781ea0fe93","RayCluster":"fail-g56b8","State":""}
{"level":"info","ts":"2026-11-03T15:06:04.556Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"09475ebd-cd31-42d2-a5cc-50781ea0fe93","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:05:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:05:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:06:07.556Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"ce54e188-0701-4a98-a4e0-fb356b5dee58","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:06:07.556Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"ce54e188-0701-4a98-a4e0-fb356b5dee58","name":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:06:07.556Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"ce54e188-0701-4a98-a4e0-fb356b5dee58","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:06:07.557Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"ce54e188-0701-4a98-a4e0-fb356b5dee58"}
{"level":"info","ts":"2026-11-03T15:06:07.557Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"ce54e188-0701-4a98-a4e0-fb356b5dee58","RayCluster":"fail-g56b8","State":""}
{"level":"info","ts":"2026-11-03T15:06:07.557Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"ce54e188-0701-4a98-a4e0-fb356b5dee58","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:05:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:05:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:06:10.558Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"cf8a981b-450f-4225-8760-47119530b1e7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:06:10.558Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"cf8a981b-450f-4225-8760-47119530b1e7","name":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:06:10.558Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"cf8a981b-450f-4225-8760-47119530b1e7","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:06:10.558Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"cf8a981b-450f-4225-8760-47119530b1e7"}
{"level":"info","ts":"2026-11-03T15:06:10.558Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"cf8a981b-450f-4225-8760-47119530b1e7","RayCluster":"fail-g56b8","State":""}
{"level":"info","ts":"2026-11-03T15:06:10.558Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"cf8a981b-450f-4225-8760-47119530b1e7","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:05:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:05:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:06:13.559Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"64d5dc4b-2171-4512-9578-545620fd77b2","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:06:13.559Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"64d5dc4b-2171-4512-9578-545620fd77b2","name":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:06:13.559Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"64d5dc4b-2171-4512-9578-545620fd77b2","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:06:13.559Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"64d5dc4b-2171-4512-9578-545620fd77b2"}
{"level":"info","ts":"2026-11-03T15:06:13.559Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"64d5dc4b-2171-4512-9578-545620fd77b2","RayCluster":"fail-g56b8","State":""}
{"level":"info","ts":"2026-11-03T15:06:13.559Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"64d5dc4b-2171-4512-9578-545620fd77b2","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:05:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:05:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:06:16.560Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"3190a4a0-89c4-4aec-90a9-70a93caa740e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:06:16.561Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"3190a4a0-89c4-4aec-90a9-70a93caa740e","name":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:06:16.561Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"3190a4a0-89c4-4aec-90a9-70a93caa740e","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:06:16.561Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"3190a4a0-89c4-4aec-90a9-70a93caa740e"}
{"level":"info","ts":"2026-11-03T15:06:16.561Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"3190a4a0-89c4-4aec-90a9-70a93caa740e","RayCluster":"fail-g56b8","State":""}
{"level":"info","ts":"2026-11-03T15:06:16.561Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"3190a4a0-89c4-4aec-90a9-70a93caa740e","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:05:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:05:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:06:19.562Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"a64407b0-141d-49dd-8180-4d09272cd31e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:06:19.562Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"a64407b0-141d-49dd-8180-4d09272cd31e","name":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:06:19.562Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"a64407b0-141d-49dd-8180-4d09272cd31e","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:06:19.562Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"a64407b0-141d-49dd-8180-4d09272cd31e"}
{"level":"info","ts":"2026-11-03T15:06:19.562Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"a64407b0-141d-49dd-8180-4d09272cd31e","RayCluster":"fail-g56b8","State":""}
{"level":"info","ts":"2026-11-03T15:06:19.562Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"a64407b0-141d-49dd-8180-4d09272cd31e","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:05:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:05:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:06:22.563Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"88981f3d-3cdc-4e86-94b2-82bdd68f4b28","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:06:22.563Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"88981f3d-3cdc-4e86-94b2-82bdd68f4b28","name":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:06:22.563Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"88981f3d-3cdc-4e86-94b2-82bdd68f4b28","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:06:22.564Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"88981f3d-3cdc-4e86-94b2-82bdd68f4b28"}
{"level":"info","ts":"2026-11-03T15:06:22.564Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"88981f3d-3cdc-4e86-94b2-82bdd68f4b28","RayCluster":"fail-g56b8","State":""}
{"level":"info","ts":"2026-11-03T15:06:22.564Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"88981f3d-3cdc-4e86-94b2-82bdd68f4b28","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:05:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:05:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:06:25.565Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"89e6b3d6-a5c7-4822-8ccf-c5b613217cc2","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:06:25.565Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"89e6b3d6-a5c7-4822-8ccf-c5b613217cc2","name":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:06:25.565Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"89e6b3d6-a5c7-4822-8ccf-c5b613217cc2","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:06:25.565Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"89e6b3d6-a5c7-4822-8ccf-c5b613217cc2"}
{"level":"info","ts":"2026-11-03T15:06:25.565Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"89e6b3d6-a5c7-4822-8ccf-c5b613217cc2","RayCluster":"fail-g56b8","State":""}
{"level":"info","ts":"2026-11-03T15:06:25.565Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"89e6b3d6-a5c7-4822-8ccf-c5b613217cc2","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:05:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:05:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:06:28.566Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"b1436f84-cd23-498e-bd01-0a3fd5b5c18b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:06:28.566Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"b1436f84-cd23-498e-bd01-0a3fd5b5c18b","name":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:06:28.566Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"b1436f84-cd23-498e-bd01-0a3fd5b5c18b","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:06:28.567Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"b1436f84-cd23-498e-bd01-0a3fd5b5c18b"}
{"level":"info","ts":"2026-11-03T15:06:28.567Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"b1436f84-cd23-498e-bd01-0a3fd5b5c18b","RayCluster":"fail-g56b8","State":""}
{"level":"info","ts":"2026-11-03T15:06:28.567Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"b1436f84-cd23-498e-bd01-0a3fd5b5c18b","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:05:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:05:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:06:30.512Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"ae659a38-d3c8-4034-b0fc-93f827b1188b"}
{"level":"info","ts":"2026-11-03T15:06:30.512Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"ae659a38-d3c8-4034-b0fc-93f827b1188b"}
{"level":"info","ts":"2026-11-03T15:06:30.512Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"ae659a38-d3c8-4034-b0fc-93f827b1188b","1 head service found":"fail-g56b8-head-svc"}
{"level":"info","ts":"2026-11-03T15:06:30.512Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"ae659a38-d3c8-4034-b0fc-93f827b1188b","Found 1 head Pod":"fail-g56b8-head-bkqfp","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:06:30.512Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"ae659a38-d3c8-4034-b0fc-93f827b1188b","head Pod":"fail-g56b8-head-bkqfp","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-g56b8-head-bkqfp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:06:30.512Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"ae659a38-d3c8-4034-b0fc-93f827b1188b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:06:30.513Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"ae659a38-d3c8-4034-b0fc-93f827b1188b","worker Pod":"fail-g56b8-small-group-worker-l77rv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-g56b8-small-group-worker-l77rv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:06:30.513Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"ae659a38-d3c8-4034-b0fc-93f827b1188b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:06:30.513Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"ae659a38-d3c8-4034-b0fc-93f827b1188b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:06:30.513Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"ae659a38-d3c8-4034-b0fc-93f827b1188b","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:06:30.513Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"ae659a38-d3c8-4034-b0fc-93f827b1188b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:06:30.513Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"ae659a38-d3c8-4034-b0fc-93f827b1188b","podName":"fail-g56b8-head-bkqfp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:06:30.513Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"ae659a38-d3c8-4034-b0fc-93f827b1188b","podName":"fail-g56b8-head-bkqfp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:06:30.513Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"ae659a38-d3c8-4034-b0fc-93f827b1188b","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:06:30.513Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"ae659a38-d3c8-4034-b0fc-93f827b1188b","seconds":300}
{"level":"info","ts":"2026-11-03T15:06:31.514Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"7400bf64-736e-493c-a608-7c353f71db27"}
{"level":"info","ts":"2026-11-03T15:06:31.514Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"7400bf64-736e-493c-a608-7c353f71db27"}
{"level":"info","ts":"2026-11-03T15:06:31.514Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"7400bf64-736e-493c-a608-7c353f71db27","1 head service found":"fail-g56b8-head-svc"}
{"level":"info","ts":"2026-11-03T15:06:31.514Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"7400bf64-736e-493c-a608-7c353f71db27","Found 1 head Pod":"fail-g56b8-head-bkqfp","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:06:31.514Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"7400bf64-736e-493c-a608-7c353f71db27","head Pod":"fail-g56b8-head-bkqfp","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-g56b8-head-bkqfp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:06:31.514Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"7400bf64-736e-493c-a608-7c353f71db27","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:06:31.515Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"7400bf64-736e-493c-a608-7c353f71db27","worker Pod":"fail-g56b8-small-group-worker-l77rv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-g56b8-small-group-worker-l77rv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:06:31.515Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"7400bf64-736e-493c-a608-7c353f71db27","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:06:31.515Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"7400bf64-736e-493c-a608-7c353f71db27","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:06:31.515Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"7400bf64-736e-493c-a608-7c353f71db27","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:06:31.515Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"7400bf64-736e-493c-a608-7c353f71db27","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:06:31.515Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"7400bf64-736e-493c-a608-7c353f71db27","podName":"fail-g56b8-head-bkqfp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:06:31.515Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"7400bf64-736e-493c-a608-7c353f71db27","podName":"fail-g56b8-head-bkqfp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:06:31.515Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"7400bf64-736e-493c-a608-7c353f71db27","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:06:31.515Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"7400bf64-736e-493c-a608-7c353f71db27","seconds":300}
{"level":"info","ts":"2026-11-03T15:06:31.567Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"ba31744a-9bce-4006-a627-26616866657f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:06:31.567Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"ba31744a-9bce-4006-a627-26616866657f","name":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:06:31.567Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"ba31744a-9bce-4006-a627-26616866657f","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:06:31.567Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"ba31744a-9bce-4006-a627-26616866657f"}
{"level":"info","ts":"2026-11-03T15:06:31.567Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"ba31744a-9bce-4006-a627-26616866657f","RayCluster":"fail-g56b8","State":""}
{"level":"info","ts":"2026-11-03T15:06:31.567Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"ba31744a-9bce-4006-a627-26616866657f","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:05:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:05:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:06:34.568Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"bc6402f0-c10a-4b50-9772-2f3dfd2e8b78","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:06:34.569Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"bc6402f0-c10a-4b50-9772-2f3dfd2e8b78","name":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:06:34.569Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"bc6402f0-c10a-4b50-9772-2f3dfd2e8b78","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:06:34.569Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"bc6402f0-c10a-4b50-9772-2f3dfd2e8b78"}
{"level":"info","ts":"2026-11-03T15:06:34.569Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"bc6402f0-c10a-4b50-9772-2f3dfd2e8b78","RayCluster":"fail-g56b8","State":""}
{"level":"info","ts":"2026-11-03T15:06:34.569Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"bc6402f0-c10a-4b50-9772-2f3dfd2e8b78","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:05:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:05:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:06:34.779Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"cdbd5510-9a84-4109-91ee-aea695bec93f"}
{"level":"info","ts":"2026-11-03T15:06:34.779Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"cdbd5510-9a84-4109-91ee-aea695bec93f"}
{"level":"info","ts":"2026-11-03T15:06:34.779Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"cdbd5510-9a84-4109-91ee-aea695bec93f","1 head service found":"fail-g56b8-head-svc"}
{"level":"info","ts":"2026-11-03T15:06:34.780Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"cdbd5510-9a84-4109-91ee-aea695bec93f","Found 1 head Pod":"fail-g56b8-head-bkqfp","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:06:34.780Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"cdbd5510-9a84-4109-91ee-aea695bec93f","head Pod":"fail-g56b8-head-bkqfp","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-g56b8-head-bkqfp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:06:34.780Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"cdbd5510-9a84-4109-91ee-aea695bec93f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:06:34.780Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"cdbd5510-9a84-4109-91ee-aea695bec93f","worker Pod":"fail-g56b8-small-group-worker-l77rv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-g56b8-small-group-worker-l77rv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:06:34.780Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"cdbd5510-9a84-4109-91ee-aea695bec93f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:06:34.780Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"cdbd5510-9a84-4109-91ee-aea695bec93f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:06:34.780Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"cdbd5510-9a84-4109-91ee-aea695bec93f","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:06:34.780Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"cdbd5510-9a84-4109-91ee-aea695bec93f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:06:34.780Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"cdbd5510-9a84-4109-91ee-aea695bec93f","podName":"fail-g56b8-head-bkqfp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:06:34.780Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"cdbd5510-9a84-4109-91ee-aea695bec93f","podName":"fail-g56b8-head-bkqfp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:06:34.780Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"cdbd5510-9a84-4109-91ee-aea695bec93f","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:06:34.780Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"cdbd5510-9a84-4109-91ee-aea695bec93f","seconds":300}
{"level":"info","ts":"2026-11-03T15:06:35.847Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"0f10b071-f92f-4124-a7d0-0324c6cc8540"}
{"level":"info","ts":"2026-11-03T15:06:35.847Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"0f10b071-f92f-4124-a7d0-0324c6cc8540"}
{"level":"info","ts":"2026-11-03T15:06:35.847Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"0f10b071-f92f-4124-a7d0-0324c6cc8540","1 head service found":"fail-g56b8-head-svc"}
{"level":"info","ts":"2026-11-03T15:06:35.847Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"0f10b071-f92f-4124-a7d0-0324c6cc8540","Found 1 head Pod":"fail-g56b8-head-bkqfp","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:06:35.847Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"0f10b071-f92f-4124-a7d0-0324c6cc8540","head Pod":"fail-g56b8-head-bkqfp","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-g56b8-head-bkqfp. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:06:35.847Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"0f10b071-f92f-4124-a7d0-0324c6cc8540","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:06:35.847Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"0f10b071-f92f-4124-a7d0-0324c6cc8540","worker Pod":"fail-g56b8-small-group-worker-l77rv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-g56b8-small-group-worker-l77rv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:06:35.847Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"0f10b071-f92f-4124-a7d0-0324c6cc8540","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:06:35.847Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"0f10b071-f92f-4124-a7d0-0324c6cc8540","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:06:35.847Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"0f10b071-f92f-4124-a7d0-0324c6cc8540","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:06:35.847Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"0f10b071-f92f-4124-a7d0-0324c6cc8540","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:06:35.847Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"0f10b071-f92f-4124-a7d0-0324c6cc8540","podName":"fail-g56b8-head-bkqfp","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:05:02Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:06:35.847Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"0f10b071-f92f-4124-a7d0-0324c6cc8540","podName":"fail-g56b8-head-bkqfp","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:05:02Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:06:35.847Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"0f10b071-f92f-4124-a7d0-0324c6cc8540","name":"fail-g56b8","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:05:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:06:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:06:35.909Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"fdad9683-cbfe-4e93-9de0-c436930b6ba3","cluster":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:06:35.909Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"efd255d8-d378-4167-aff0-ff65882a8c89","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:06:35.909Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"efd255d8-d378-4167-aff0-ff65882a8c89","name":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:06:35.909Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"efd255d8-d378-4167-aff0-ff65882a8c89","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:06:35.909Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"efd255d8-d378-4167-aff0-ff65882a8c89"}
{"level":"info","ts":"2026-11-03T15:06:35.909Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"efd255d8-d378-4167-aff0-ff65882a8c89","RayCluster":"fail-g56b8","State":""}
{"level":"info","ts":"2026-11-03T15:06:35.909Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"efd255d8-d378-4167-aff0-ff65882a8c89","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:05:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:06:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:06:35.909Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"efd255d8-d378-4167-aff0-ff65882a8c89","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:06:35.971Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"6a564d8a-3bba-42c4-adbe-fdc9f56b619b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:06:35.971Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"6a564d8a-3bba-42c4-adbe-fdc9f56b619b","name":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:06:35.971Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"6a564d8a-3bba-42c4-adbe-fdc9f56b619b","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:06:35.971Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"6a564d8a-3bba-42c4-adbe-fdc9f56b619b"}
{"level":"info","ts":"2026-11-03T15:06:35.971Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"6a564d8a-3bba-42c4-adbe-fdc9f56b619b","RayCluster":"fail-g56b8","State":""}
{"level":"info","ts":"2026-11-03T15:06:35.971Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"6a564d8a-3bba-42c4-adbe-fdc9f56b619b","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:06:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:06:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:06:37.910Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"0496dea3-94ed-4752-9dc1-171ecb46afef"}
{"level":"info","ts":"2026-11-03T15:06:37.910Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"0496dea3-94ed-4752-9dc1-171ecb46afef"}
{"level":"info","ts":"2026-11-03T15:06:37.910Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"0496dea3-94ed-4752-9dc1-171ecb46afef","1 head service found":"fail-g56b8-head-svc"}
{"level":"info","ts":"2026-11-03T15:06:37.910Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"0496dea3-94ed-4752-9dc1-171ecb46afef","Found 1 head Pod":"fail-g56b8-head-bkqfp","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:06:37.910Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"0496dea3-94ed-4752-9dc1-171ecb46afef","head Pod":"fail-g56b8-head-bkqfp","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-g56b8-head-bkqfp. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:06:37.910Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"0496dea3-94ed-4752-9dc1-171ecb46afef","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:06:37.910Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"0496dea3-94ed-4752-9dc1-171ecb46afef","worker Pod":"fail-g56b8-small-group-worker-l77rv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-g56b8-small-group-worker-l77rv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:06:37.910Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"0496dea3-94ed-4752-9dc1-171ecb46afef","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:06:37.910Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"0496dea3-94ed-4752-9dc1-171ecb46afef","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:06:37.910Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"0496dea3-94ed-4752-9dc1-171ecb46afef","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:06:37.910Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"0496dea3-94ed-4752-9dc1-171ecb46afef","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:06:37.910Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"0496dea3-94ed-4752-9dc1-171ecb46afef","podName":"fail-g56b8-head-bkqfp","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:05:02Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:06:37.910Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"0496dea3-94ed-4752-9dc1-171ecb46afef","podName":"fail-g56b8-head-bkqfp","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:05:02Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:06:37.910Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"0496dea3-94ed-4752-9dc1-171ecb46afef","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:06:37.910Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"0496dea3-94ed-4752-9dc1-171ecb46afef","seconds":300}
{"level":"info","ts":"2026-11-03T15:06:38.972Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"af72f439-67be-4386-9eca-96b875a3d256","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:06:38.972Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"af72f439-67be-4386-9eca-96b875a3d256","name":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:06:38.972Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"af72f439-67be-4386-9eca-96b875a3d256","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:06:38.972Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"af72f439-67be-4386-9eca-96b875a3d256"}
{"level":"info","ts":"2026-11-03T15:06:38.972Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"af72f439-67be-4386-9eca-96b875a3d256","RayCluster":"fail-g56b8","State":""}
{"level":"info","ts":"2026-11-03T15:06:38.972Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"af72f439-67be-4386-9eca-96b875a3d256","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:06:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:06:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:06:40.568Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"193dc268-58a7-4aa5-ba26-1bee2d93a213"}
{"level":"info","ts":"2026-11-03T15:06:40.568Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"193dc268-58a7-4aa5-ba26-1bee2d93a213"}
{"level":"info","ts":"2026-11-03T15:06:40.568Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"193dc268-58a7-4aa5-ba26-1bee2d93a213","1 head service found":"fail-g56b8-head-svc"}
{"level":"info","ts":"2026-11-03T15:06:40.568Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"193dc268-58a7-4aa5-ba26-1bee2d93a213","Found 1 head Pod":"fail-g56b8-head-bkqfp","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:06:40.568Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"193dc268-58a7-4aa5-ba26-1bee2d93a213","head Pod":"fail-g56b8-head-bkqfp","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-g56b8-head-bkqfp. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:06:40.568Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"193dc268-58a7-4aa5-ba26-1bee2d93a213","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:06:40.568Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"193dc268-58a7-4aa5-ba26-1bee2d93a213","worker Pod":"fail-g56b8-small-group-worker-l77rv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-g56b8-small-group-worker-l77rv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:06:40.568Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"193dc268-58a7-4aa5-ba26-1bee2d93a213","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:06:40.568Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"193dc268-58a7-4aa5-ba26-1bee2d93a213","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:06:40.568Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"193dc268-58a7-4aa5-ba26-1bee2d93a213","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:06:40.568Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"193dc268-58a7-4aa5-ba26-1bee2d93a213","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:06:40.568Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"193dc268-58a7-4aa5-ba26-1bee2d93a213","podName":"fail-g56b8-head-bkqfp","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:05:02Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:06:40.568Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"193dc268-58a7-4aa5-ba26-1bee2d93a213","podName":"fail-g56b8-head-bkqfp","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:05:02Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:06:40.568Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"193dc268-58a7-4aa5-ba26-1bee2d93a213","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:06:40.568Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"193dc268-58a7-4aa5-ba26-1bee2d93a213","seconds":300}
{"level":"info","ts":"2026-11-03T15:06:41.581Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"dff0d991-c202-4e76-9b0c-037bc2ce3149"}
{"level":"info","ts":"2026-11-03T15:06:41.581Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"dff0d991-c202-4e76-9b0c-037bc2ce3149"}
{"level":"info","ts":"2026-11-03T15:06:41.581Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"dff0d991-c202-4e76-9b0c-037bc2ce3149","1 head service found":"fail-g56b8-head-svc"}
{"level":"info","ts":"2026-11-03T15:06:41.581Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"dff0d991-c202-4e76-9b0c-037bc2ce3149","Found 1 head Pod":"fail-g56b8-head-bkqfp","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:06:41.581Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"dff0d991-c202-4e76-9b0c-037bc2ce3149","head Pod":"fail-g56b8-head-bkqfp","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-g56b8-head-bkqfp. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:06:41.581Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"dff0d991-c202-4e76-9b0c-037bc2ce3149","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:06:41.581Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"dff0d991-c202-4e76-9b0c-037bc2ce3149","worker Pod":"fail-g56b8-small-group-worker-l77rv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-g56b8-small-group-worker-l77rv. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:06:41.581Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"dff0d991-c202-4e76-9b0c-037bc2ce3149","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:06:41.581Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"dff0d991-c202-4e76-9b0c-037bc2ce3149","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:06:41.581Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"dff0d991-c202-4e76-9b0c-037bc2ce3149","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:06:41.581Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"dff0d991-c202-4e76-9b0c-037bc2ce3149","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:06:41.581Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"dff0d991-c202-4e76-9b0c-037bc2ce3149","podName":"fail-g56b8-head-bkqfp","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:05:02Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:06:41.581Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"dff0d991-c202-4e76-9b0c-037bc2ce3149","podName":"fail-g56b8-head-bkqfp","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:05:02Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:06:41.582Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"dff0d991-c202-4e76-9b0c-037bc2ce3149","name":"fail-g56b8","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:06:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:06:41Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:06:41.642Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"75836d7f-ff9f-423e-b275-6ebe1a64d66d","cluster":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:06:41.643Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"0b10d47c-1d10-4a42-842e-aa3520e55602","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:06:41.643Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"0b10d47c-1d10-4a42-842e-aa3520e55602","name":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:06:41.643Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"0b10d47c-1d10-4a42-842e-aa3520e55602","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:06:41.643Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"0b10d47c-1d10-4a42-842e-aa3520e55602"}
{"level":"info","ts":"2026-11-03T15:06:41.643Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"0b10d47c-1d10-4a42-842e-aa3520e55602","RayCluster":"fail-g56b8","State":""}
{"level":"info","ts":"2026-11-03T15:06:41.643Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"0b10d47c-1d10-4a42-842e-aa3520e55602","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:06:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:06:41Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:06:41.643Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"0b10d47c-1d10-4a42-842e-aa3520e55602","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:06:41.703Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"aaaf738b-b61e-457f-bb29-37a7dd761f0e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:06:41.703Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"aaaf738b-b61e-457f-bb29-37a7dd761f0e","name":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:06:41.703Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"aaaf738b-b61e-457f-bb29-37a7dd761f0e","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:06:41.703Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"aaaf738b-b61e-457f-bb29-37a7dd761f0e"}
{"level":"info","ts":"2026-11-03T15:06:41.703Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"aaaf738b-b61e-457f-bb29-37a7dd761f0e","RayCluster":"fail-g56b8","State":""}
{"level":"info","ts":"2026-11-03T15:06:41.703Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"aaaf738b-b61e-457f-bb29-37a7dd761f0e","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:06:41Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:06:41Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:06:43.643Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"e9938902-a456-4b22-85f5-710b97e67d8a"}
{"level":"info","ts":"2026-11-03T15:06:43.643Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"e9938902-a456-4b22-85f5-710b97e67d8a"}
{"level":"info","ts":"2026-11-03T15:06:43.643Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"e9938902-a456-4b22-85f5-710b97e67d8a","1 head service found":"fail-g56b8-head-svc"}
{"level":"info","ts":"2026-11-03T15:06:43.643Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"e9938902-a456-4b22-85f5-710b97e67d8a","Found 1 head Pod":"fail-g56b8-head-bkqfp","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:06:43.643Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"e9938902-a456-4b22-85f5-710b97e67d8a","head Pod":"fail-g56b8-head-bkqfp","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-g56b8-head-bkqfp. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:06:43.643Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"e9938902-a456-4b22-85f5-710b97e67d8a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:06:43.643Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"e9938902-a456-4b22-85f5-710b97e67d8a","worker Pod":"fail-g56b8-small-group-worker-l77rv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-g56b8-small-group-worker-l77rv. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:06:43.643Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"e9938902-a456-4b22-85f5-710b97e67d8a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:06:43.643Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"e9938902-a456-4b22-85f5-710b97e67d8a","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:06:43.643Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"e9938902-a456-4b22-85f5-710b97e67d8a","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:06:43.643Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"e9938902-a456-4b22-85f5-710b97e67d8a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:06:43.643Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"e9938902-a456-4b22-85f5-710b97e67d8a","podName":"fail-g56b8-head-bkqfp","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:05:02Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:06:43.643Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"e9938902-a456-4b22-85f5-710b97e67d8a","podName":"fail-g56b8-head-bkqfp","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:05:02Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:06:43.643Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"e9938902-a456-4b22-85f5-710b97e67d8a","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:06:43.643Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"e9938902-a456-4b22-85f5-710b97e67d8a","seconds":300}
{"level":"info","ts":"2026-11-03T15:06:44.704Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"b03b551b-ff06-4cb1-b264-27fd2aa8b256","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:06:44.704Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"b03b551b-ff06-4cb1-b264-27fd2aa8b256","name":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:06:44.704Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"b03b551b-ff06-4cb1-b264-27fd2aa8b256","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:06:44.704Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"b03b551b-ff06-4cb1-b264-27fd2aa8b256"}
{"level":"info","ts":"2026-11-03T15:06:44.704Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"b03b551b-ff06-4cb1-b264-27fd2aa8b256","RayCluster":"fail-g56b8","State":""}
{"level":"info","ts":"2026-11-03T15:06:44.704Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"b03b551b-ff06-4cb1-b264-27fd2aa8b256","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:06:41Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:06:41Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:06:46.847Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"dcbe2d4f-ed33-4867-b20a-c4339c7e9f56"}
{"level":"info","ts":"2026-11-03T15:06:46.847Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"dcbe2d4f-ed33-4867-b20a-c4339c7e9f56"}
{"level":"info","ts":"2026-11-03T15:06:46.847Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"dcbe2d4f-ed33-4867-b20a-c4339c7e9f56","1 head service found":"fail-g56b8-head-svc"}
{"level":"info","ts":"2026-11-03T15:06:46.847Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"dcbe2d4f-ed33-4867-b20a-c4339c7e9f56","Found 1 head Pod":"fail-g56b8-head-bkqfp","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:06:46.847Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"dcbe2d4f-ed33-4867-b20a-c4339c7e9f56","head Pod":"fail-g56b8-head-bkqfp","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-g56b8-head-bkqfp. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:06:46.847Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"dcbe2d4f-ed33-4867-b20a-c4339c7e9f56","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:06:46.847Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"dcbe2d4f-ed33-4867-b20a-c4339c7e9f56","worker Pod":"fail-g56b8-small-group-worker-l77rv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-g56b8-small-group-worker-l77rv. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:06:46.847Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"dcbe2d4f-ed33-4867-b20a-c4339c7e9f56","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:06:46.847Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"dcbe2d4f-ed33-4867-b20a-c4339c7e9f56","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:06:46.847Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"dcbe2d4f-ed33-4867-b20a-c4339c7e9f56","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:06:46.847Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"dcbe2d4f-ed33-4867-b20a-c4339c7e9f56","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:06:46.848Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"dcbe2d4f-ed33-4867-b20a-c4339c7e9f56","podName":"fail-g56b8-small-group-worker-l77rv","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:05:02Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:06:46.848Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"dcbe2d4f-ed33-4867-b20a-c4339c7e9f56","podName":"fail-g56b8-small-group-worker-l77rv","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:05:02Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:06:46.848Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"dcbe2d4f-ed33-4867-b20a-c4339c7e9f56","name":"fail-g56b8","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:06:41Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:06:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:06:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:06:46.909Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"0228231c-c611-4065-807d-7e2f87bb97df","cluster":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:06:46.909Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"5a94f38e-d15b-4aa3-b05c-811f015a6946","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:06:46.909Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"5a94f38e-d15b-4aa3-b05c-811f015a6946","name":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:06:46.909Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"5a94f38e-d15b-4aa3-b05c-811f015a6946","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:06:46.909Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"5a94f38e-d15b-4aa3-b05c-811f015a6946"}
{"level":"info","ts":"2026-11-03T15:06:46.909Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"5a94f38e-d15b-4aa3-b05c-811f015a6946","RayCluster":"fail-g56b8","State":""}
{"level":"info","ts":"2026-11-03T15:06:46.910Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"5a94f38e-d15b-4aa3-b05c-811f015a6946","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:06:41Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:06:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:06:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:06:46.910Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"5a94f38e-d15b-4aa3-b05c-811f015a6946","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:06:46.970Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"7f6646b1-f6b2-44a6-a48a-1116a431bd51","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:06:46.970Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"7f6646b1-f6b2-44a6-a48a-1116a431bd51","name":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:06:46.970Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"7f6646b1-f6b2-44a6-a48a-1116a431bd51","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:06:46.970Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"7f6646b1-f6b2-44a6-a48a-1116a431bd51"}
{"level":"info","ts":"2026-11-03T15:06:46.970Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"7f6646b1-f6b2-44a6-a48a-1116a431bd51","RayCluster":"fail-g56b8","State":""}
{"level":"info","ts":"2026-11-03T15:06:46.970Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"7f6646b1-f6b2-44a6-a48a-1116a431bd51","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:06:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:06:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:06:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:06:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:06:48.909Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"d6a2bbed-522f-4a2a-a42f-77d166ff6efc"}
{"level":"info","ts":"2026-11-03T15:06:48.910Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"d6a2bbed-522f-4a2a-a42f-77d166ff6efc"}
{"level":"info","ts":"2026-11-03T15:06:48.910Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"d6a2bbed-522f-4a2a-a42f-77d166ff6efc","1 head service found":"fail-g56b8-head-svc"}
{"level":"info","ts":"2026-11-03T15:06:48.910Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"d6a2bbed-522f-4a2a-a42f-77d166ff6efc","Found 1 head Pod":"fail-g56b8-head-bkqfp","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:06:48.910Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"d6a2bbed-522f-4a2a-a42f-77d166ff6efc","head Pod":"fail-g56b8-head-bkqfp","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-g56b8-head-bkqfp. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:06:48.910Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"d6a2bbed-522f-4a2a-a42f-77d166ff6efc","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:06:48.910Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"d6a2bbed-522f-4a2a-a42f-77d166ff6efc","worker Pod":"fail-g56b8-small-group-worker-l77rv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-g56b8-small-group-worker-l77rv. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:06:48.910Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"d6a2bbed-522f-4a2a-a42f-77d166ff6efc","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:06:48.910Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"d6a2bbed-522f-4a2a-a42f-77d166ff6efc","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:06:48.910Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"d6a2bbed-522f-4a2a-a42f-77d166ff6efc","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:06:48.910Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"d6a2bbed-522f-4a2a-a42f-77d166ff6efc","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:06:48.910Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"d6a2bbed-522f-4a2a-a42f-77d166ff6efc","podName":"fail-g56b8-small-group-worker-l77rv","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:05:02Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:06:48.910Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"d6a2bbed-522f-4a2a-a42f-77d166ff6efc","podName":"fail-g56b8-small-group-worker-l77rv","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:05:02Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:06:48.910Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"d6a2bbed-522f-4a2a-a42f-77d166ff6efc","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:06:48.910Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"d6a2bbed-522f-4a2a-a42f-77d166ff6efc","seconds":300}
{"level":"info","ts":"2026-11-03T15:06:49.971Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"7c766339-85a6-4850-964c-38a263bb1c14","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:06:49.971Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"7c766339-85a6-4850-964c-38a263bb1c14","name":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:06:49.971Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"7c766339-85a6-4850-964c-38a263bb1c14","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:06:49.972Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"7c766339-85a6-4850-964c-38a263bb1c14"}
{"level":"info","ts":"2026-11-03T15:06:49.972Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"7c766339-85a6-4850-964c-38a263bb1c14","RayCluster":"fail-g56b8","State":""}
{"level":"info","ts":"2026-11-03T15:06:49.972Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"7c766339-85a6-4850-964c-38a263bb1c14","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:06:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:06:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:06:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:06:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:06:52.592Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"8089fbb1-31de-4c0d-bf27-a2ab65bc50d4"}
{"level":"info","ts":"2026-11-03T15:06:52.593Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"8089fbb1-31de-4c0d-bf27-a2ab65bc50d4"}
{"level":"info","ts":"2026-11-03T15:06:52.593Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"8089fbb1-31de-4c0d-bf27-a2ab65bc50d4","1 head service found":"fail-g56b8-head-svc"}
{"level":"info","ts":"2026-11-03T15:06:52.593Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"8089fbb1-31de-4c0d-bf27-a2ab65bc50d4","Found 1 head Pod":"fail-g56b8-head-bkqfp","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:06:52.593Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"8089fbb1-31de-4c0d-bf27-a2ab65bc50d4","head Pod":"fail-g56b8-head-bkqfp","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-g56b8-head-bkqfp. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:06:52.593Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"8089fbb1-31de-4c0d-bf27-a2ab65bc50d4","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:06:52.593Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"8089fbb1-31de-4c0d-bf27-a2ab65bc50d4","worker Pod":"fail-g56b8-small-group-worker-l77rv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-g56b8-small-group-worker-l77rv. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:06:52.593Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"8089fbb1-31de-4c0d-bf27-a2ab65bc50d4","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:06:52.593Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"8089fbb1-31de-4c0d-bf27-a2ab65bc50d4","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:06:52.593Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"8089fbb1-31de-4c0d-bf27-a2ab65bc50d4","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:06:52.593Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"8089fbb1-31de-4c0d-bf27-a2ab65bc50d4","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:06:52.593Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"8089fbb1-31de-4c0d-bf27-a2ab65bc50d4","name":"fail-g56b8","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:06:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:06:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:06:52Z","stateTransitionTimes":{"ready":"2026-11-03T15:06:52Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:06:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:06:52Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:06:52.655Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"c85db8fe-2962-4a63-93ef-cbb2f686909d","cluster":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:06:52.655Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"af234fa6-d61b-4da1-af00-753d568c32ff","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:06:52.655Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"af234fa6-d61b-4da1-af00-753d568c32ff","name":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:06:52.655Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"af234fa6-d61b-4da1-af00-753d568c32ff","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:06:52.655Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"af234fa6-d61b-4da1-af00-753d568c32ff"}
{"level":"info","ts":"2026-11-03T15:06:52.655Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"af234fa6-d61b-4da1-af00-753d568c32ff","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:06:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:06:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","dashboardURL":"fail-g56b8-head-svc.test-ns-mvz6h.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:06:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:06:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:06:52.655Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"af234fa6-d61b-4da1-af00-753d568c32ff","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-11-03T15:06:52.715Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"360a4e03-747f-435d-8c8b-05c976f4c047","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:06:52.715Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"360a4e03-747f-435d-8c8b-05c976f4c047","name":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:06:52.715Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"360a4e03-747f-435d-8c8b-05c976f4c047","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:06:52.715Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"360a4e03-747f-435d-8c8b-05c976f4c047"}
{"level":"info","ts":"2026-11-03T15:06:52.715Z","logger":"controllers.RayJob","msg":"Create a Kubernetes Service","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"360a4e03-747f-435d-8c8b-05c976f4c047","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:06:52.786Z","logger":"controllers.RayJob","msg":"The Ray job was not found. Submit a Ray job via an HTTP request.","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"360a4e03-747f-435d-8c8b-05c976f4c047","JobId":"fail-9m8gw"}
{"level":"info","ts":"2026-11-03T15:06:53.632Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"0740fd79-d906-4ead-ae33-76c7994d444b","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:06:53.632Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"0740fd79-d906-4ead-ae33-76c7994d444b","name":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:06:53.632Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"0740fd79-d906-4ead-ae33-76c7994d444b","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:06:53.632Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"0740fd79-d906-4ead-ae33-76c7994d444b"}
{"level":"info","ts":"2026-11-03T15:06:53.632Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"0740fd79-d906-4ead-ae33-76c7994d444b","rayCluster":"fail-g56b8","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:06:53.635Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"0740fd79-d906-4ead-ae33-76c7994d444b","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","dashboardURL":"fail-g56b8-head-svc.test-ns-mvz6h.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:06:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:06:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:06:53Z"},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","dashboardURL":"fail-g56b8-head-svc.test-ns-mvz6h.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:06:52Z","stateTransitionTimes":{"ready":"2026-11-03T15:06:52Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:06:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:06:52Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:06:53.635Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"0740fd79-d906-4ead-ae33-76c7994d444b","old JobStatus":"","new JobStatus":"PENDING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-11-03T15:06:53.698Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"c494eaad-888b-4905-aa57-49baebdaa5f4","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:06:53.698Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"c494eaad-888b-4905-aa57-49baebdaa5f4","name":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:06:53.698Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"c494eaad-888b-4905-aa57-49baebdaa5f4","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:06:53.698Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"c494eaad-888b-4905-aa57-49baebdaa5f4"}
{"level":"info","ts":"2026-11-03T15:06:53.698Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"c494eaad-888b-4905-aa57-49baebdaa5f4","rayCluster":"fail-g56b8","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:06:53.701Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"c494eaad-888b-4905-aa57-49baebdaa5f4","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:06:53Z"},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","dashboardURL":"fail-g56b8-head-svc.test-ns-mvz6h.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:06:52Z","stateTransitionTimes":{"ready":"2026-11-03T15:06:52Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:06:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:06:52Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:06:53Z"},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","dashboardURL":"fail-g56b8-head-svc.test-ns-mvz6h.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:06:52Z","stateTransitionTimes":{"ready":"2026-11-03T15:06:52Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:06:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:06:52Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:06:54.655Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"35ac1974-3d2d-494b-a6b6-52495ca86a99"}
{"level":"info","ts":"2026-11-03T15:06:54.655Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"35ac1974-3d2d-494b-a6b6-52495ca86a99"}
{"level":"info","ts":"2026-11-03T15:06:54.655Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"35ac1974-3d2d-494b-a6b6-52495ca86a99","1 head service found":"fail-g56b8-head-svc"}
{"level":"info","ts":"2026-11-03T15:06:54.655Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"35ac1974-3d2d-494b-a6b6-52495ca86a99","Found 1 head Pod":"fail-g56b8-head-bkqfp","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:06:54.655Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"35ac1974-3d2d-494b-a6b6-52495ca86a99","head Pod":"fail-g56b8-head-bkqfp","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-g56b8-head-bkqfp. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:06:54.655Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"35ac1974-3d2d-494b-a6b6-52495ca86a99","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:06:54.655Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"35ac1974-3d2d-494b-a6b6-52495ca86a99","worker Pod":"fail-g56b8-small-group-worker-l77rv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-g56b8-small-group-worker-l77rv. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:06:54.655Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"35ac1974-3d2d-494b-a6b6-52495ca86a99","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:06:54.655Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"35ac1974-3d2d-494b-a6b6-52495ca86a99","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:06:54.655Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"35ac1974-3d2d-494b-a6b6-52495ca86a99","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:06:54.655Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"35ac1974-3d2d-494b-a6b6-52495ca86a99","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:06:54.655Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"35ac1974-3d2d-494b-a6b6-52495ca86a99","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:06:54.655Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"35ac1974-3d2d-494b-a6b6-52495ca86a99","seconds":300}
{"level":"info","ts":"2026-11-03T15:06:56.702Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"1dc5fb23-0428-4fec-812d-c825830f312c","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:06:56.702Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"1dc5fb23-0428-4fec-812d-c825830f312c","name":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:06:56.702Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"1dc5fb23-0428-4fec-812d-c825830f312c","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:06:56.703Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"1dc5fb23-0428-4fec-812d-c825830f312c"}
{"level":"info","ts":"2026-11-03T15:06:56.703Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"1dc5fb23-0428-4fec-812d-c825830f312c","rayCluster":"fail-g56b8","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:06:56.706Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"1dc5fb23-0428-4fec-812d-c825830f312c","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:06:53Z"},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","dashboardURL":"fail-g56b8-head-svc.test-ns-mvz6h.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:06:52Z","stateTransitionTimes":{"ready":"2026-11-03T15:06:52Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:06:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:06:52Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:06:53Z","endTime":"2026-11-03T15:06:54Z"},"jobId":"fail-9m8gw","rayClusterName":"fail-g56b8","dashboardURL":"fail-g56b8-head-svc.test-ns-mvz6h.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Failed","reason":"AppFailed","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-11-03 15:06:53,359\tINFO job_manager.py:583 -- Runtime env is setting up.\nRunning entrypoint for job fail-9m8gw: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-11-03T15:04:58Z","succeeded":0,"failed":1,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:06:52Z","stateTransitionTimes":{"ready":"2026-11-03T15:06:52Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.32","serviceIP":"10.132.0.32","podName":"fail-g56b8-head-bkqfp","serviceName":"fail-g56b8-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:06:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:06:52Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:04:59Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:04:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:06:56.706Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"1dc5fb23-0428-4fec-812d-c825830f312c","old JobStatus":"PENDING","new JobStatus":"FAILED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Failed"}
{"level":"info","ts":"2026-11-03T15:06:56.802Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"6c73f9ce-2420-49f6-bcf0-a2445b34ca80","JobStatus":"FAILED","JobDeploymentStatus":"Failed","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:06:57.030Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"7deba1c0-9e67-484c-9126-a418768d6197","DeletionTimestamp":"2026-11-03 15:06:56 +0000 UTC"}
{"level":"info","ts":"2026-11-03T15:06:57.030Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"7deba1c0-9e67-484c-9126-a418768d6197","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-11-03T15:06:57.093Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"42194ea0-dbca-4fae-9b3e-90c1452affd0"}
{"level":"info","ts":"2026-11-03T15:06:57.109Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"a9e5da07-36c7-4a21-a213-1a07081e1fb4"}
{"level":"info","ts":"2026-11-03T15:06:57.115Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"fail-g56b8","reconcileID":"246a42c3-974a-4bd5-ad6e-3829701d5d00","namespacedName":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:06:57.115Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"fail-g56b8","reconcileID":"246a42c3-974a-4bd5-ad6e-3829701d5d00","cluster":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:06:57.115Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"fail-g56b8","reconcileID":"246a42c3-974a-4bd5-ad6e-3829701d5d00","cluster":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:06:57.115Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"fail-g56b8","reconcileID":"246a42c3-974a-4bd5-ad6e-3829701d5d00","namespace":"test-ns-mvz6h","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-11-03T15:06:57.115Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"reconcileID":"1786fbed-14c3-4042-aae1-6fa947cd1edb"}
{"level":"info","ts":"2026-11-03T15:06:57.116Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"bf57bc36-967b-4006-afa5-c58541236a41"}
{"level":"info","ts":"2026-11-03T15:06:57.168Z","logger":"authentication-controller","msg":"Deleted shared ReferenceGrant (last cluster with auth)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"fail-g56b8","reconcileID":"246a42c3-974a-4bd5-ad6e-3829701d5d00","name":"kuberay-gateway-access","namespace":"test-ns-mvz6h"}
{"level":"info","ts":"2026-11-03T15:06:57.220Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"fail-g56b8","reconcileID":"246a42c3-974a-4bd5-ad6e-3829701d5d00","name":"test-ns-mvz6h-fail-g56b8","namespace":"opendatahub"}
{"level":"info","ts":"2026-11-03T15:06:57.272Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"fail-g56b8","reconcileID":"246a42c3-974a-4bd5-ad6e-3829701d5d00","configMap":"kube-rbac-proxy-config-fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:06:57.324Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"fail-g56b8","reconcileID":"246a42c3-974a-4bd5-ad6e-3829701d5d00","serviceAccount":"fail-g56b8-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-11-03T15:06:57.489Z","logger":"raycluster-resource","msg":"default","name":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:06:57.489Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"fail-g56b8","namespace":"test-ns-mvz6h"}
{"level":"info","ts":"2026-11-03T15:06:57.654Z","logger":"raycluster-resource","msg":"validate update","name":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:06:57.709Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"fail-g56b8","reconcileID":"246a42c3-974a-4bd5-ad6e-3829701d5d00","cluster":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:06:57.709Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"fail-g56b8","reconcileID":"f1d9a0cf-9031-4d26-ba71-6d6c6d6f689e","namespacedName":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:06:57.709Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"fail-g56b8","reconcileID":"f1d9a0cf-9031-4d26-ba71-6d6c6d6f689e","cluster":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:06:57.709Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"fail-g56b8","reconcileID":"f1d9a0cf-9031-4d26-ba71-6d6c6d6f689e","cluster":"fail-g56b8"}
{"level":"info","ts":"2026-11-03T15:06:57.709Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"fail-g56b8","reconcileID":"f1d9a0cf-9031-4d26-ba71-6d6c6d6f689e","namespace":"test-ns-mvz6h","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-11-03T15:06:57.709Z","logger":"authentication-controller","msg":"ReferenceGrant already deleted","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"fail-g56b8","reconcileID":"f1d9a0cf-9031-4d26-ba71-6d6c6d6f689e","name":"kuberay-gateway-access"}
{"level":"info","ts":"2026-11-03T15:06:57.709Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"fail","namespace":"test-ns-mvz6h"},"reconcileID":"57c57ae5-39df-4c94-9b6d-1b401c34356d"}
{"level":"error","ts":"2026-11-03T15:06:57.760Z","logger":"authentication-controller","msg":"Failed to remove finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"fail-g56b8","reconcileID":"f1d9a0cf-9031-4d26-ba71-6d6c6d6f689e","error":"rayclusters.ray.io \"fail-g56b8\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleDeletion\n\t/workspace/controllers/ray/authentication_controller.go:195\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:06:57.760Z","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"fail-g56b8","reconcileID":"f1d9a0cf-9031-4d26-ba71-6d6c6d6f689e"}
{"level":"error","ts":"2026-11-03T15:06:57.760Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"fail-g56b8","reconcileID":"f1d9a0cf-9031-4d26-ba71-6d6c6d6f689e","error":"rayclusters.ray.io \"fail-g56b8\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:06:57.760Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"fail-g56b8","reconcileID":"15d64e03-4379-4e32-9ab3-ad90129ef712","namespacedName":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:06:57.760Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"fail-g56b8","reconcileID":"15d64e03-4379-4e32-9ab3-ad90129ef712"}
{"level":"info","ts":"2026-11-03T15:06:57.760Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"fail-g56b8","reconcileID":"15d64e03-4379-4e32-9ab3-ad90129ef712","namespace":"test-ns-mvz6h","authClusters":0}
{"level":"info","ts":"2026-11-03T15:06:57.760Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-g56b8","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"fail-g56b8","reconcileID":"15d64e03-4379-4e32-9ab3-ad90129ef712","namespace":"test-ns-mvz6h"}
{"level":"info","ts":"2026-11-03T15:06:58.086Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"07b7d288-c1d7-4467-a337-27c44fad2eca","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:06:58.086Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"07b7d288-c1d7-4467-a337-27c44fad2eca","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-11-03T15:06:58.146Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"07b7d288-c1d7-4467-a337-27c44fad2eca"}
{"level":"info","ts":"2026-11-03T15:06:58.146Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"07b7d288-c1d7-4467-a337-27c44fad2eca","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-11-03T15:06:58.146Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"07b7d288-c1d7-4467-a337-27c44fad2eca","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:06:58.146Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"07b7d288-c1d7-4467-a337-27c44fad2eca","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:06:58.205Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"2809629f-80c0-4739-a596-6e36a21cc655","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:06:58.205Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"2809629f-80c0-4739-a596-6e36a21cc655"}
{"level":"info","ts":"2026-11-03T15:06:58.205Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"2809629f-80c0-4739-a596-6e36a21cc655","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-11-03T15:06:58.205Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"2809629f-80c0-4739-a596-6e36a21cc655","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-hpb5h","rayClusterName":"stop-5wsj2","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:06:58.205Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"2809629f-80c0-4739-a596-6e36a21cc655","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:06:58.258Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"2809629f-80c0-4739-a596-6e36a21cc655","error":"Operation cannot be fulfilled on rayjobs.ray.io \"stop\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-11-03T15:06:58.258Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"2809629f-80c0-4739-a596-6e36a21cc655"}
{"level":"error","ts":"2026-11-03T15:06:58.258Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"2809629f-80c0-4739-a596-6e36a21cc655","error":"Operation cannot be fulfilled on rayjobs.ray.io \"stop\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:06:58.258Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"cb252abc-6779-40df-beae-feaa3ed662f1","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:06:58.258Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"cb252abc-6779-40df-beae-feaa3ed662f1","name":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:06:58.258Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"cb252abc-6779-40df-beae-feaa3ed662f1","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:06:58.258Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"cb252abc-6779-40df-beae-feaa3ed662f1","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:06:58.310Z","logger":"raycluster-resource","msg":"default","name":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:06:58.310Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"stop-254kg","namespace":"test-ns-mvz6h"}
{"level":"info","ts":"2026-11-03T15:06:58.371Z","logger":"raycluster-resource","msg":"validate create","name":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:06:58.426Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"cb252abc-6779-40df-beae-feaa3ed662f1","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:06:58.426Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"cb252abc-6779-40df-beae-feaa3ed662f1"}
{"level":"info","ts":"2026-11-03T15:06:58.426Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"cb252abc-6779-40df-beae-feaa3ed662f1","RayCluster":"stop-254kg","State":""}
{"level":"info","ts":"2026-11-03T15:06:58.426Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"cb252abc-6779-40df-beae-feaa3ed662f1","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:06:58.427Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"f1dd646c-4dfc-4901-8be6-3e66bf28aed7"}
{"level":"info","ts":"2026-11-03T15:06:58.427Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"f1dd646c-4dfc-4901-8be6-3e66bf28aed7"}
{"level":"info","ts":"2026-11-03T15:06:58.427Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"7274bec2-4789-4c37-aaf1-d68dfea68a50","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:06:58.427Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"7274bec2-4789-4c37-aaf1-d68dfea68a50","name":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:06:58.427Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"7274bec2-4789-4c37-aaf1-d68dfea68a50","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:06:58.427Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"stop-254kg","reconcileID":"24419ad2-5be0-4a10-a17e-c408d21efa37","namespacedName":{"name":"stop-254kg","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:06:58.427Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"7274bec2-4789-4c37-aaf1-d68dfea68a50"}
{"level":"info","ts":"2026-11-03T15:06:58.427Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"7274bec2-4789-4c37-aaf1-d68dfea68a50","RayCluster":"stop-254kg","State":""}
{"level":"info","ts":"2026-11-03T15:06:58.427Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"7274bec2-4789-4c37-aaf1-d68dfea68a50","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:06:58.427Z","logger":"controllers.NetworkPolicy","msg":"Reconciling NetworkPolicies for RayCluster","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"639df555-dd3a-44a8-823b-0b77f40e196f","cluster":"stop-254kg","namespace":"test-ns-mvz6h"}
{"level":"info","ts":"2026-11-03T15:06:58.427Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"fe4d3fad-3c5b-4119-967a-321204156f11","cluster":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:06:58.427Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"stop-254kg","reconcileID":"24419ad2-5be0-4a10-a17e-c408d21efa37","mode":"IntegratedOAuth","cluster":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:06:58.427Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"stop-254kg","reconcileID":"24419ad2-5be0-4a10-a17e-c408d21efa37","cluster":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:06:58.427Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"stop-254kg","reconcileID":"24419ad2-5be0-4a10-a17e-c408d21efa37","cluster":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:06:58.480Z","logger":"raycluster-resource","msg":"default","name":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:06:58.480Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"stop-254kg","namespace":"test-ns-mvz6h"}
{"level":"info","ts":"2026-11-03T15:06:58.481Z","logger":"controllers.NetworkPolicy","msg":"Successfully created NetworkPolicy","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"639df555-dd3a-44a8-823b-0b77f40e196f","name":"stop-254kg-head"}
{"level":"info","ts":"2026-11-03T15:06:58.481Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"f1dd646c-4dfc-4901-8be6-3e66bf28aed7","name":"stop-254kg-head-svc"}
{"level":"info","ts":"2026-11-03T15:06:58.482Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"f1dd646c-4dfc-4901-8be6-3e66bf28aed7"}
{"level":"info","ts":"2026-11-03T15:06:58.532Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"f1dd646c-4dfc-4901-8be6-3e66bf28aed7","error":"certificate ray-head-cert-stop-254kg not found: certificates.cert-manager.io \"ray-head-cert-stop-254kg\" not found"}
{"level":"error","ts":"2026-11-03T15:06:58.532Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"f1dd646c-4dfc-4901-8be6-3e66bf28aed7","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-stop-254kg not found: certificates.cert-manager.io \"ray-head-cert-stop-254kg\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:06:58.532Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"f1dd646c-4dfc-4901-8be6-3e66bf28aed7","error":"mTLS secrets not ready: certificate ray-head-cert-stop-254kg not found: certificates.cert-manager.io \"ray-head-cert-stop-254kg\" not found"}
{"level":"info","ts":"2026-11-03T15:06:58.532Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"f1dd646c-4dfc-4901-8be6-3e66bf28aed7","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"stop-254kg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:06:58.532Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"f1dd646c-4dfc-4901-8be6-3e66bf28aed7","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"stop-254kg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:06:58.532Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"f1dd646c-4dfc-4901-8be6-3e66bf28aed7","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"stop-254kg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:06:58.532Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"f1dd646c-4dfc-4901-8be6-3e66bf28aed7","name":"stop-254kg","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:06:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:06:58.533Z","logger":"controllers.NetworkPolicy","msg":"Successfully created NetworkPolicy","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"639df555-dd3a-44a8-823b-0b77f40e196f","name":"stop-254kg-workers-small-group"}
{"level":"info","ts":"2026-11-03T15:06:58.533Z","logger":"controllers.NetworkPolicy","msg":"Successfully reconciled NetworkPolicies for RayCluster","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"639df555-dd3a-44a8-823b-0b77f40e196f","cluster":"stop-254kg","namespace":"test-ns-mvz6h"}
{"level":"info","ts":"2026-11-03T15:06:58.533Z","logger":"controllers.NetworkPolicy","msg":"Reconciling NetworkPolicies for RayCluster","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"d6dd5ef2-aa81-4f13-833c-be09203801f4","cluster":"stop-254kg","namespace":"test-ns-mvz6h"}
{"level":"info","ts":"2026-11-03T15:06:58.534Z","logger":"raycluster-resource","msg":"validate update","name":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:06:58.590Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"b83a85aa-e956-4659-a6e6-5e3eb742e03e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:06:58.590Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"b83a85aa-e956-4659-a6e6-5e3eb742e03e","name":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:06:58.590Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"b83a85aa-e956-4659-a6e6-5e3eb742e03e","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:06:58.590Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"b83a85aa-e956-4659-a6e6-5e3eb742e03e"}
{"level":"info","ts":"2026-11-03T15:06:58.590Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"b83a85aa-e956-4659-a6e6-5e3eb742e03e","RayCluster":"stop-254kg","State":""}
{"level":"info","ts":"2026-11-03T15:06:58.590Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"b83a85aa-e956-4659-a6e6-5e3eb742e03e","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:06:58.594Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"f1dd646c-4dfc-4901-8be6-3e66bf28aed7","name":"stop-254kg","error":"Operation cannot be fulfilled on rayclusters.ray.io \"stop-254kg\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-mvz6h","name":"stop-254kg"}}
{"level":"info","ts":"2026-11-03T15:06:58.594Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"f1dd646c-4dfc-4901-8be6-3e66bf28aed7"}
{"level":"error","ts":"2026-11-03T15:06:58.594Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"f1dd646c-4dfc-4901-8be6-3e66bf28aed7","error":"mTLS secrets not ready: certificate ray-head-cert-stop-254kg not found: certificates.cert-manager.io \"ray-head-cert-stop-254kg\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:06:58.594Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"419c254c-5491-41f5-933b-33263da56c36"}
{"level":"info","ts":"2026-11-03T15:06:58.594Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"419c254c-5491-41f5-933b-33263da56c36"}
{"level":"info","ts":"2026-11-03T15:06:58.594Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"419c254c-5491-41f5-933b-33263da56c36","1 head service found":"stop-254kg-head-svc"}
{"level":"info","ts":"2026-11-03T15:06:58.594Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"419c254c-5491-41f5-933b-33263da56c36"}
{"level":"info","ts":"2026-11-03T15:06:58.645Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"419c254c-5491-41f5-933b-33263da56c36","error":"certificate ray-head-cert-stop-254kg not found: certificates.cert-manager.io \"ray-head-cert-stop-254kg\" not found"}
{"level":"error","ts":"2026-11-03T15:06:58.645Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"419c254c-5491-41f5-933b-33263da56c36","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-stop-254kg not found: certificates.cert-manager.io \"ray-head-cert-stop-254kg\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:06:58.645Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"419c254c-5491-41f5-933b-33263da56c36","error":"mTLS secrets not ready: certificate ray-head-cert-stop-254kg not found: certificates.cert-manager.io \"ray-head-cert-stop-254kg\" not found"}
{"level":"info","ts":"2026-11-03T15:06:58.645Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"419c254c-5491-41f5-933b-33263da56c36","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"stop-254kg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:06:58.646Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"419c254c-5491-41f5-933b-33263da56c36","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"stop-254kg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:06:58.646Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"419c254c-5491-41f5-933b-33263da56c36","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"stop-254kg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:06:58.646Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"419c254c-5491-41f5-933b-33263da56c36","name":"stop-254kg","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:06:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:06:58.646Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"stop-254kg","reconcileID":"24419ad2-5be0-4a10-a17e-c408d21efa37","name":"stop-254kg-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-11-03T15:06:58.647Z","logger":"controllers.NetworkPolicy","msg":"Successfully reconciled NetworkPolicies for RayCluster","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"d6dd5ef2-aa81-4f13-833c-be09203801f4","cluster":"stop-254kg","namespace":"test-ns-mvz6h"}
{"level":"info","ts":"2026-11-03T15:06:58.700Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"stop-254kg","reconcileID":"24419ad2-5be0-4a10-a17e-c408d21efa37","name":"kuberay-gateway-access","namespace":"test-ns-mvz6h","operation":"created"}
{"level":"info","ts":"2026-11-03T15:06:58.705Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"419c254c-5491-41f5-933b-33263da56c36"}
{"level":"error","ts":"2026-11-03T15:06:58.705Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"419c254c-5491-41f5-933b-33263da56c36","error":"mTLS secrets not ready: certificate ray-head-cert-stop-254kg not found: certificates.cert-manager.io \"ray-head-cert-stop-254kg\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:06:58.706Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"426a1202-5463-4e4e-a0e6-4dc3a3613cd5","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:06:58.706Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"426a1202-5463-4e4e-a0e6-4dc3a3613cd5","name":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:06:58.706Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"426a1202-5463-4e4e-a0e6-4dc3a3613cd5","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:06:58.706Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"426a1202-5463-4e4e-a0e6-4dc3a3613cd5"}
{"level":"info","ts":"2026-11-03T15:06:58.706Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"426a1202-5463-4e4e-a0e6-4dc3a3613cd5","RayCluster":"stop-254kg","State":""}
{"level":"info","ts":"2026-11-03T15:06:58.706Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"426a1202-5463-4e4e-a0e6-4dc3a3613cd5","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:06:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:06:58.706Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"426a1202-5463-4e4e-a0e6-4dc3a3613cd5","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:06:58.715Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"01a0f37a-3076-4848-a837-0f6fda8176b7"}
{"level":"info","ts":"2026-11-03T15:06:58.715Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"01a0f37a-3076-4848-a837-0f6fda8176b7"}
{"level":"info","ts":"2026-11-03T15:06:58.715Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"01a0f37a-3076-4848-a837-0f6fda8176b7","1 head service found":"stop-254kg-head-svc"}
{"level":"info","ts":"2026-11-03T15:06:58.715Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"01a0f37a-3076-4848-a837-0f6fda8176b7"}
{"level":"info","ts":"2026-11-03T15:06:58.758Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"stop-254kg","reconcileID":"24419ad2-5be0-4a10-a17e-c408d21efa37","name":"test-ns-mvz6h-stop-254kg","operation":"created"}
{"level":"info","ts":"2026-11-03T15:06:58.765Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"01a0f37a-3076-4848-a837-0f6fda8176b7","error":"certificate ray-head-cert-stop-254kg not found: certificates.cert-manager.io \"ray-head-cert-stop-254kg\" not found"}
{"level":"error","ts":"2026-11-03T15:06:58.765Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"01a0f37a-3076-4848-a837-0f6fda8176b7","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-stop-254kg not found: certificates.cert-manager.io \"ray-head-cert-stop-254kg\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:06:58.765Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"01a0f37a-3076-4848-a837-0f6fda8176b7","error":"mTLS secrets not ready: certificate ray-head-cert-stop-254kg not found: certificates.cert-manager.io \"ray-head-cert-stop-254kg\" not found"}
{"level":"info","ts":"2026-11-03T15:06:58.765Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"01a0f37a-3076-4848-a837-0f6fda8176b7","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"stop-254kg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:06:58.765Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"01a0f37a-3076-4848-a837-0f6fda8176b7","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"stop-254kg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:06:58.765Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"01a0f37a-3076-4848-a837-0f6fda8176b7","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"stop-254kg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:06:58.765Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"01a0f37a-3076-4848-a837-0f6fda8176b7"}
{"level":"error","ts":"2026-11-03T15:06:58.765Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"01a0f37a-3076-4848-a837-0f6fda8176b7","error":"mTLS secrets not ready: certificate ray-head-cert-stop-254kg not found: certificates.cert-manager.io \"ray-head-cert-stop-254kg\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:06:58.766Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"e9c31489-8430-472a-ab75-e571d07b1ba3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:06:58.766Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"e9c31489-8430-472a-ab75-e571d07b1ba3","name":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:06:58.766Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"e9c31489-8430-472a-ab75-e571d07b1ba3","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:06:58.766Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"e9c31489-8430-472a-ab75-e571d07b1ba3"}
{"level":"info","ts":"2026-11-03T15:06:58.766Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"e9c31489-8430-472a-ab75-e571d07b1ba3","RayCluster":"stop-254kg","State":""}
{"level":"info","ts":"2026-11-03T15:06:58.766Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"e9c31489-8430-472a-ab75-e571d07b1ba3","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:06:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:06:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:06:58.785Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"42968317-69dd-4b4e-9394-319bef836088"}
{"level":"info","ts":"2026-11-03T15:06:58.785Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"42968317-69dd-4b4e-9394-319bef836088"}
{"level":"info","ts":"2026-11-03T15:06:58.785Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"42968317-69dd-4b4e-9394-319bef836088","1 head service found":"stop-254kg-head-svc"}
{"level":"info","ts":"2026-11-03T15:06:58.785Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"42968317-69dd-4b4e-9394-319bef836088"}
{"level":"info","ts":"2026-11-03T15:06:58.815Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"stop-254kg","reconcileID":"24419ad2-5be0-4a10-a17e-c408d21efa37","name":"kube-rbac-proxy-config-stop-254kg","operation":"created"}
{"level":"info","ts":"2026-11-03T15:06:58.838Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"42968317-69dd-4b4e-9394-319bef836088","error":"certificate ray-head-cert-stop-254kg not found: certificates.cert-manager.io \"ray-head-cert-stop-254kg\" not found"}
{"level":"error","ts":"2026-11-03T15:06:58.838Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"42968317-69dd-4b4e-9394-319bef836088","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-stop-254kg not found: certificates.cert-manager.io \"ray-head-cert-stop-254kg\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:06:58.838Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"42968317-69dd-4b4e-9394-319bef836088","error":"mTLS secrets not ready: certificate ray-head-cert-stop-254kg not found: certificates.cert-manager.io \"ray-head-cert-stop-254kg\" not found"}
{"level":"info","ts":"2026-11-03T15:06:58.838Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"42968317-69dd-4b4e-9394-319bef836088","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"stop-254kg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:06:58.838Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"42968317-69dd-4b4e-9394-319bef836088","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"stop-254kg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:06:58.838Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"42968317-69dd-4b4e-9394-319bef836088","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"stop-254kg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:06:58.838Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"42968317-69dd-4b4e-9394-319bef836088"}
{"level":"error","ts":"2026-11-03T15:06:58.838Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"42968317-69dd-4b4e-9394-319bef836088","error":"mTLS secrets not ready: certificate ray-head-cert-stop-254kg not found: certificates.cert-manager.io \"ray-head-cert-stop-254kg\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-03T15:06:58.868Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"stop-254kg","reconcileID":"24419ad2-5be0-4a10-a17e-c408d21efa37","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"stop-254kg\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:167\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:06:58.868Z","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"stop-254kg","reconcileID":"24419ad2-5be0-4a10-a17e-c408d21efa37"}
{"level":"error","ts":"2026-11-03T15:06:58.868Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"stop-254kg","reconcileID":"24419ad2-5be0-4a10-a17e-c408d21efa37","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"stop-254kg\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:06:58.868Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"stop-254kg","reconcileID":"93acbd0a-1254-42d0-b966-19f1370957ef","namespacedName":{"name":"stop-254kg","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:06:58.868Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"stop-254kg","reconcileID":"93acbd0a-1254-42d0-b966-19f1370957ef","mode":"IntegratedOAuth","cluster":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:06:58.868Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"stop-254kg","reconcileID":"93acbd0a-1254-42d0-b966-19f1370957ef","cluster":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:06:58.879Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"a6e9e6f8-433b-405f-81dc-2e7695d304e6"}
{"level":"info","ts":"2026-11-03T15:06:58.879Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"a6e9e6f8-433b-405f-81dc-2e7695d304e6"}
{"level":"info","ts":"2026-11-03T15:06:58.879Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"a6e9e6f8-433b-405f-81dc-2e7695d304e6","1 head service found":"stop-254kg-head-svc"}
{"level":"info","ts":"2026-11-03T15:06:58.879Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"a6e9e6f8-433b-405f-81dc-2e7695d304e6"}
{"level":"info","ts":"2026-11-03T15:06:58.928Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"stop-254kg","reconcileID":"93acbd0a-1254-42d0-b966-19f1370957ef","cluster":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:06:58.929Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"a6e9e6f8-433b-405f-81dc-2e7695d304e6","error":"certificate ray-head-cert-stop-254kg not found: certificates.cert-manager.io \"ray-head-cert-stop-254kg\" not found"}
{"level":"error","ts":"2026-11-03T15:06:58.929Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"a6e9e6f8-433b-405f-81dc-2e7695d304e6","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-stop-254kg not found: certificates.cert-manager.io \"ray-head-cert-stop-254kg\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:06:58.929Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"a6e9e6f8-433b-405f-81dc-2e7695d304e6","error":"mTLS secrets not ready: certificate ray-head-cert-stop-254kg not found: certificates.cert-manager.io \"ray-head-cert-stop-254kg\" not found"}
{"level":"info","ts":"2026-11-03T15:06:58.929Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"a6e9e6f8-433b-405f-81dc-2e7695d304e6","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"stop-254kg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:06:58.929Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"a6e9e6f8-433b-405f-81dc-2e7695d304e6","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"stop-254kg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:06:58.929Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"a6e9e6f8-433b-405f-81dc-2e7695d304e6","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"stop-254kg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:06:58.929Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"a6e9e6f8-433b-405f-81dc-2e7695d304e6"}
{"level":"error","ts":"2026-11-03T15:06:58.929Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"a6e9e6f8-433b-405f-81dc-2e7695d304e6","error":"mTLS secrets not ready: certificate ray-head-cert-stop-254kg not found: certificates.cert-manager.io \"ray-head-cert-stop-254kg\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:06:58.930Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"aec01e06-9acd-4bb1-9c80-6e5eb5ea9b0b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:06:58.930Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"aec01e06-9acd-4bb1-9c80-6e5eb5ea9b0b","name":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:06:58.930Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"aec01e06-9acd-4bb1-9c80-6e5eb5ea9b0b","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:06:58.930Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"aec01e06-9acd-4bb1-9c80-6e5eb5ea9b0b"}
{"level":"info","ts":"2026-11-03T15:06:58.930Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"aec01e06-9acd-4bb1-9c80-6e5eb5ea9b0b","RayCluster":"stop-254kg","State":""}
{"level":"info","ts":"2026-11-03T15:06:58.930Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"aec01e06-9acd-4bb1-9c80-6e5eb5ea9b0b","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:06:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:06:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:06:58.930Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"aec01e06-9acd-4bb1-9c80-6e5eb5ea9b0b","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:06:58.991Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"01ac73b7-8b32-44cf-9710-9e13c699ccd1","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:06:58.991Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"01ac73b7-8b32-44cf-9710-9e13c699ccd1","name":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:06:58.991Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"01ac73b7-8b32-44cf-9710-9e13c699ccd1","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:06:58.991Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"01ac73b7-8b32-44cf-9710-9e13c699ccd1"}
{"level":"info","ts":"2026-11-03T15:06:58.991Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"01ac73b7-8b32-44cf-9710-9e13c699ccd1","RayCluster":"stop-254kg","State":""}
{"level":"info","ts":"2026-11-03T15:06:58.991Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"01ac73b7-8b32-44cf-9710-9e13c699ccd1","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:06:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:06:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:06:59.009Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"22c1be63-cf49-4cea-a739-518515e507c0"}
{"level":"info","ts":"2026-11-03T15:06:59.009Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"22c1be63-cf49-4cea-a739-518515e507c0"}
{"level":"info","ts":"2026-11-03T15:06:59.009Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"22c1be63-cf49-4cea-a739-518515e507c0","1 head service found":"stop-254kg-head-svc"}
{"level":"info","ts":"2026-11-03T15:06:59.010Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"22c1be63-cf49-4cea-a739-518515e507c0"}
{"level":"info","ts":"2026-11-03T15:06:59.059Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"22c1be63-cf49-4cea-a739-518515e507c0","error":"certificate ray-head-cert-stop-254kg not found: certificates.cert-manager.io \"ray-head-cert-stop-254kg\" not found"}
{"level":"error","ts":"2026-11-03T15:06:59.059Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"22c1be63-cf49-4cea-a739-518515e507c0","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-stop-254kg not found: certificates.cert-manager.io \"ray-head-cert-stop-254kg\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:06:59.059Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"22c1be63-cf49-4cea-a739-518515e507c0","error":"mTLS secrets not ready: certificate ray-head-cert-stop-254kg not found: certificates.cert-manager.io \"ray-head-cert-stop-254kg\" not found"}
{"level":"info","ts":"2026-11-03T15:06:59.059Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"22c1be63-cf49-4cea-a739-518515e507c0","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"stop-254kg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:06:59.059Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"22c1be63-cf49-4cea-a739-518515e507c0","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"stop-254kg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:06:59.059Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"22c1be63-cf49-4cea-a739-518515e507c0","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"stop-254kg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:06:59.059Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"22c1be63-cf49-4cea-a739-518515e507c0"}
{"level":"error","ts":"2026-11-03T15:06:59.059Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"22c1be63-cf49-4cea-a739-518515e507c0","error":"mTLS secrets not ready: certificate ray-head-cert-stop-254kg not found: certificates.cert-manager.io \"ray-head-cert-stop-254kg\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:06:59.220Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"5cbd45de-8b87-4db0-bffa-f8dce63b923e"}
{"level":"info","ts":"2026-11-03T15:06:59.220Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"5cbd45de-8b87-4db0-bffa-f8dce63b923e"}
{"level":"info","ts":"2026-11-03T15:06:59.220Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"5cbd45de-8b87-4db0-bffa-f8dce63b923e","1 head service found":"stop-254kg-head-svc"}
{"level":"info","ts":"2026-11-03T15:06:59.220Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"5cbd45de-8b87-4db0-bffa-f8dce63b923e"}
{"level":"info","ts":"2026-11-03T15:06:59.270Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"5cbd45de-8b87-4db0-bffa-f8dce63b923e","error":"certificate ray-head-cert-stop-254kg is not ready at current generation 1"}
{"level":"error","ts":"2026-11-03T15:06:59.270Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"5cbd45de-8b87-4db0-bffa-f8dce63b923e","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-stop-254kg is not ready at current generation 1","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:06:59.270Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"5cbd45de-8b87-4db0-bffa-f8dce63b923e","error":"mTLS secrets not ready: certificate ray-head-cert-stop-254kg is not ready at current generation 1"}
{"level":"info","ts":"2026-11-03T15:06:59.270Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"5cbd45de-8b87-4db0-bffa-f8dce63b923e","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"stop-254kg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:06:59.270Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"5cbd45de-8b87-4db0-bffa-f8dce63b923e","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"stop-254kg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:06:59.270Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"5cbd45de-8b87-4db0-bffa-f8dce63b923e","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"stop-254kg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:06:59.270Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"5cbd45de-8b87-4db0-bffa-f8dce63b923e"}
{"level":"error","ts":"2026-11-03T15:06:59.270Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"5cbd45de-8b87-4db0-bffa-f8dce63b923e","error":"mTLS secrets not ready: certificate ray-head-cert-stop-254kg is not ready at current generation 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:06:59.378Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"fe4d3fad-3c5b-4119-967a-321204156f11","certificate":"ray-head-cert-stop-254kg"}
{"level":"info","ts":"2026-11-03T15:06:59.378Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"fe4d3fad-3c5b-4119-967a-321204156f11"}
{"level":"info","ts":"2026-11-03T15:06:59.378Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"f0207afe-da97-4f48-9a2a-9723f1e54db2","cluster":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:06:59.591Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"0e00e949-d089-4bf9-8ac7-eb430c727341"}
{"level":"info","ts":"2026-11-03T15:06:59.591Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"0e00e949-d089-4bf9-8ac7-eb430c727341"}
{"level":"info","ts":"2026-11-03T15:06:59.591Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"0e00e949-d089-4bf9-8ac7-eb430c727341","1 head service found":"stop-254kg-head-svc"}
{"level":"info","ts":"2026-11-03T15:06:59.591Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"0e00e949-d089-4bf9-8ac7-eb430c727341"}
{"level":"info","ts":"2026-11-03T15:06:59.643Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"0e00e949-d089-4bf9-8ac7-eb430c727341","error":"certificate ray-head-cert-stop-254kg is not ready at current generation 1"}
{"level":"error","ts":"2026-11-03T15:06:59.643Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"0e00e949-d089-4bf9-8ac7-eb430c727341","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-stop-254kg is not ready at current generation 1","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:06:59.644Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"0e00e949-d089-4bf9-8ac7-eb430c727341","error":"mTLS secrets not ready: certificate ray-head-cert-stop-254kg is not ready at current generation 1"}
{"level":"info","ts":"2026-11-03T15:06:59.644Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"0e00e949-d089-4bf9-8ac7-eb430c727341","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"stop-254kg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:06:59.644Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"0e00e949-d089-4bf9-8ac7-eb430c727341","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"stop-254kg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:06:59.644Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"0e00e949-d089-4bf9-8ac7-eb430c727341","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"stop-254kg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:06:59.644Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"0e00e949-d089-4bf9-8ac7-eb430c727341"}
{"level":"error","ts":"2026-11-03T15:06:59.644Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"0e00e949-d089-4bf9-8ac7-eb430c727341","error":"mTLS secrets not ready: certificate ray-head-cert-stop-254kg is not ready at current generation 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:06:59.831Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"f0207afe-da97-4f48-9a2a-9723f1e54db2","certificate":"ray-head-cert-stop-254kg"}
{"level":"info","ts":"2026-11-03T15:06:59.831Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"f0207afe-da97-4f48-9a2a-9723f1e54db2"}
{"level":"info","ts":"2026-11-03T15:07:00.284Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"38d08081-c68e-4df0-a738-30ac92fb9c51"}
{"level":"info","ts":"2026-11-03T15:07:00.285Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"38d08081-c68e-4df0-a738-30ac92fb9c51"}
{"level":"info","ts":"2026-11-03T15:07:00.285Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"38d08081-c68e-4df0-a738-30ac92fb9c51","1 head service found":"stop-254kg-head-svc"}
{"level":"info","ts":"2026-11-03T15:07:00.285Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"38d08081-c68e-4df0-a738-30ac92fb9c51"}
{"level":"info","ts":"2026-11-03T15:07:00.335Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"38d08081-c68e-4df0-a738-30ac92fb9c51","error":"certificate ray-head-cert-stop-254kg is not ready at current generation 1"}
{"level":"error","ts":"2026-11-03T15:07:00.335Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"38d08081-c68e-4df0-a738-30ac92fb9c51","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-stop-254kg is not ready at current generation 1","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:07:00.335Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"38d08081-c68e-4df0-a738-30ac92fb9c51","error":"mTLS secrets not ready: certificate ray-head-cert-stop-254kg is not ready at current generation 1"}
{"level":"info","ts":"2026-11-03T15:07:00.335Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"38d08081-c68e-4df0-a738-30ac92fb9c51","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"stop-254kg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:07:00.335Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"38d08081-c68e-4df0-a738-30ac92fb9c51","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"stop-254kg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:07:00.335Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"38d08081-c68e-4df0-a738-30ac92fb9c51","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"stop-254kg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:07:00.335Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"38d08081-c68e-4df0-a738-30ac92fb9c51"}
{"level":"error","ts":"2026-11-03T15:07:00.335Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"38d08081-c68e-4df0-a738-30ac92fb9c51","error":"mTLS secrets not ready: certificate ray-head-cert-stop-254kg is not ready at current generation 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:07:01.616Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"456db779-47ea-426e-8a96-87c904d583f4"}
{"level":"info","ts":"2026-11-03T15:07:01.616Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"456db779-47ea-426e-8a96-87c904d583f4"}
{"level":"info","ts":"2026-11-03T15:07:01.616Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"456db779-47ea-426e-8a96-87c904d583f4","1 head service found":"stop-254kg-head-svc"}
{"level":"info","ts":"2026-11-03T15:07:01.616Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"456db779-47ea-426e-8a96-87c904d583f4"}
{"level":"info","ts":"2026-11-03T15:07:01.666Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"456db779-47ea-426e-8a96-87c904d583f4","error":"certificate ray-head-cert-stop-254kg is not ready at current generation 1"}
{"level":"error","ts":"2026-11-03T15:07:01.666Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"456db779-47ea-426e-8a96-87c904d583f4","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-stop-254kg is not ready at current generation 1","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:07:01.666Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"456db779-47ea-426e-8a96-87c904d583f4","error":"mTLS secrets not ready: certificate ray-head-cert-stop-254kg is not ready at current generation 1"}
{"level":"info","ts":"2026-11-03T15:07:01.666Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"456db779-47ea-426e-8a96-87c904d583f4","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"stop-254kg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:07:01.666Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"456db779-47ea-426e-8a96-87c904d583f4","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"stop-254kg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:07:01.666Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"456db779-47ea-426e-8a96-87c904d583f4","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"stop-254kg","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:07:01.666Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"456db779-47ea-426e-8a96-87c904d583f4"}
{"level":"error","ts":"2026-11-03T15:07:01.666Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"456db779-47ea-426e-8a96-87c904d583f4","error":"mTLS secrets not ready: certificate ray-head-cert-stop-254kg is not ready at current generation 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:07:01.992Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"e5e65892-9315-4e4c-a74a-9a9bf88e144a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:07:01.992Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"e5e65892-9315-4e4c-a74a-9a9bf88e144a","name":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:07:01.992Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"e5e65892-9315-4e4c-a74a-9a9bf88e144a","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:07:01.992Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"e5e65892-9315-4e4c-a74a-9a9bf88e144a"}
{"level":"info","ts":"2026-11-03T15:07:01.992Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"e5e65892-9315-4e4c-a74a-9a9bf88e144a","RayCluster":"stop-254kg","State":""}
{"level":"info","ts":"2026-11-03T15:07:01.992Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"e5e65892-9315-4e4c-a74a-9a9bf88e144a","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:06:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:06:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:07:04.227Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"32be8278-dcdf-4ace-94f2-bc8b674f00b3"}
{"level":"info","ts":"2026-11-03T15:07:04.227Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"32be8278-dcdf-4ace-94f2-bc8b674f00b3"}
{"level":"info","ts":"2026-11-03T15:07:04.227Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"32be8278-dcdf-4ace-94f2-bc8b674f00b3","1 head service found":"stop-254kg-head-svc"}
{"level":"info","ts":"2026-11-03T15:07:04.227Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"32be8278-dcdf-4ace-94f2-bc8b674f00b3"}
{"level":"info","ts":"2026-11-03T15:07:04.328Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"32be8278-dcdf-4ace-94f2-bc8b674f00b3","mode":"IntegratedOAuth","cluster":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:07:04.328Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"32be8278-dcdf-4ace-94f2-bc8b674f00b3","cluster":"stop-254kg","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-11-03T15:07:04.328Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"32be8278-dcdf-4ace-94f2-bc8b674f00b3","cluster":"stop-254kg","authType":"IntegratedOAuth","serviceAccount":"stop-254kg-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-11-03T15:07:04.328Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"32be8278-dcdf-4ace-94f2-bc8b674f00b3","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"stop-254kg","ray.io/group":"headgroup","ray.io/identifier":"stop-254kg-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-11-03T15:07:04.328Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"32be8278-dcdf-4ace-94f2-bc8b674f00b3","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-11-03T15:07:04.328Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"32be8278-dcdf-4ace-94f2-bc8b674f00b3","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-11-03T15:07:04.328Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"32be8278-dcdf-4ace-94f2-bc8b674f00b3","rayNodeType":"head","generatedCmd":"ulimit -n ${RAY_START_ULIMIT_OPEN_FILES:-65536}; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-11-03T15:07:04.328Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"32be8278-dcdf-4ace-94f2-bc8b674f00b3","enabled":true}
{"level":"info","ts":"2026-11-03T15:07:04.447Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"32be8278-dcdf-4ace-94f2-bc8b674f00b3","name":"stop-254kg-head-4nd85"}
{"level":"info","ts":"2026-11-03T15:07:04.447Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"32be8278-dcdf-4ace-94f2-bc8b674f00b3","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:07:04.447Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"32be8278-dcdf-4ace-94f2-bc8b674f00b3","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:07:04.447Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"32be8278-dcdf-4ace-94f2-bc8b674f00b3","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-11-03T15:07:04.447Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"32be8278-dcdf-4ace-94f2-bc8b674f00b3","group":"small-group","indices":{}}
{"level":"info","ts":"2026-11-03T15:07:04.447Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"32be8278-dcdf-4ace-94f2-bc8b674f00b3","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-11-03T15:07:04.547Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"32be8278-dcdf-4ace-94f2-bc8b674f00b3","creating worker for group":"small-group","index":0,"total":1,"replicaIndex":0}
{"level":"info","ts":"2026-11-03T15:07:04.547Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"32be8278-dcdf-4ace-94f2-bc8b674f00b3","nodeType":"worker","rayStartParams":{"address":"stop-254kg-head-svc.test-ns-mvz6h.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-11-03T15:07:04.547Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"32be8278-dcdf-4ace-94f2-bc8b674f00b3","rayStartCmd":"ray start  --address=stop-254kg-head-svc.test-ns-mvz6h.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-11-03T15:07:04.547Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"32be8278-dcdf-4ace-94f2-bc8b674f00b3","rayNodeType":"worker","generatedCmd":"ulimit -n ${RAY_START_ULIMIT_OPEN_FILES:-65536}; ray start  --address=stop-254kg-head-svc.test-ns-mvz6h.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-11-03T15:07:04.547Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"32be8278-dcdf-4ace-94f2-bc8b674f00b3","enabled":true}
{"level":"info","ts":"2026-11-03T15:07:04.609Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"32be8278-dcdf-4ace-94f2-bc8b674f00b3","name":"stop-254kg-small-group-worker-nhg8h"}
{"level":"info","ts":"2026-11-03T15:07:04.609Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"32be8278-dcdf-4ace-94f2-bc8b674f00b3","podName":"stop-254kg-head-4nd85","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:07:04.609Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"32be8278-dcdf-4ace-94f2-bc8b674f00b3","podName":"stop-254kg-head-4nd85","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:07:04.609Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"32be8278-dcdf-4ace-94f2-bc8b674f00b3","name":"stop-254kg","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:06:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:07:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:07:04.670Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"f8dfd3f0-e27f-43d0-a847-228c1274b290"}
{"level":"info","ts":"2026-11-03T15:07:04.670Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"f8dfd3f0-e27f-43d0-a847-228c1274b290"}
{"level":"info","ts":"2026-11-03T15:07:04.670Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"f8dfd3f0-e27f-43d0-a847-228c1274b290","1 head service found":"stop-254kg-head-svc"}
{"level":"info","ts":"2026-11-03T15:07:04.670Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"76636ae4-4e78-4680-b1e5-379139d15d7e","cluster":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:07:04.670Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"75577b89-1fb7-40da-8fef-ac6f3268940c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:07:04.670Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"75577b89-1fb7-40da-8fef-ac6f3268940c","name":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:07:04.670Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"75577b89-1fb7-40da-8fef-ac6f3268940c","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:07:04.670Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"f8dfd3f0-e27f-43d0-a847-228c1274b290","Found 1 head Pod":"stop-254kg-head-4nd85","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:07:04.670Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"f8dfd3f0-e27f-43d0-a847-228c1274b290","head Pod":"stop-254kg-head-4nd85","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-254kg-head-4nd85. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:07:04.670Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"f8dfd3f0-e27f-43d0-a847-228c1274b290","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:07:04.670Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"f8dfd3f0-e27f-43d0-a847-228c1274b290","worker Pod":"stop-254kg-small-group-worker-nhg8h","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-254kg-small-group-worker-nhg8h. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:07:04.670Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"f8dfd3f0-e27f-43d0-a847-228c1274b290","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:07:04.670Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"f8dfd3f0-e27f-43d0-a847-228c1274b290","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:07:04.670Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"f8dfd3f0-e27f-43d0-a847-228c1274b290","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:07:04.670Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"f8dfd3f0-e27f-43d0-a847-228c1274b290","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:07:04.671Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"f8dfd3f0-e27f-43d0-a847-228c1274b290","podName":"stop-254kg-head-4nd85","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:07:04.671Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"75577b89-1fb7-40da-8fef-ac6f3268940c"}
{"level":"info","ts":"2026-11-03T15:07:04.671Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"75577b89-1fb7-40da-8fef-ac6f3268940c","RayCluster":"stop-254kg","State":""}
{"level":"info","ts":"2026-11-03T15:07:04.671Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"f8dfd3f0-e27f-43d0-a847-228c1274b290","podName":"stop-254kg-head-4nd85","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:07:04.671Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"75577b89-1fb7-40da-8fef-ac6f3268940c","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:06:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:07:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:07:04.671Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"75577b89-1fb7-40da-8fef-ac6f3268940c","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:07:04.671Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"f8dfd3f0-e27f-43d0-a847-228c1274b290","name":"stop-254kg","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:06:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:07:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:07:04.725Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"f8dfd3f0-e27f-43d0-a847-228c1274b290","name":"stop-254kg","error":"Operation cannot be fulfilled on rayclusters.ray.io \"stop-254kg\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-mvz6h","name":"stop-254kg"}}
{"level":"info","ts":"2026-11-03T15:07:04.725Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"f8dfd3f0-e27f-43d0-a847-228c1274b290"}
{"level":"error","ts":"2026-11-03T15:07:04.725Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"f8dfd3f0-e27f-43d0-a847-228c1274b290","error":"Operation cannot be fulfilled on rayclusters.ray.io \"stop-254kg\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:07:04.725Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"0dc4bc9f-7044-47c0-86e1-70b77547b431"}
{"level":"info","ts":"2026-11-03T15:07:04.725Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"0dc4bc9f-7044-47c0-86e1-70b77547b431"}
{"level":"info","ts":"2026-11-03T15:07:04.725Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"0dc4bc9f-7044-47c0-86e1-70b77547b431","1 head service found":"stop-254kg-head-svc"}
{"level":"info","ts":"2026-11-03T15:07:04.725Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"0dc4bc9f-7044-47c0-86e1-70b77547b431","Found 1 head Pod":"stop-254kg-head-4nd85","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:07:04.725Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"0dc4bc9f-7044-47c0-86e1-70b77547b431","head Pod":"stop-254kg-head-4nd85","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-254kg-head-4nd85. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:07:04.725Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"0dc4bc9f-7044-47c0-86e1-70b77547b431","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:07:04.725Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"0dc4bc9f-7044-47c0-86e1-70b77547b431","worker Pod":"stop-254kg-small-group-worker-nhg8h","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-254kg-small-group-worker-nhg8h. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:07:04.725Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"0dc4bc9f-7044-47c0-86e1-70b77547b431","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:07:04.725Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"0dc4bc9f-7044-47c0-86e1-70b77547b431","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:07:04.725Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"0dc4bc9f-7044-47c0-86e1-70b77547b431","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:07:04.725Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"0dc4bc9f-7044-47c0-86e1-70b77547b431","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:07:04.725Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"0dc4bc9f-7044-47c0-86e1-70b77547b431","podName":"stop-254kg-head-4nd85","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:07:04.725Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"0dc4bc9f-7044-47c0-86e1-70b77547b431","podName":"stop-254kg-head-4nd85","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:07:04.726Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"0dc4bc9f-7044-47c0-86e1-70b77547b431","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:07:04.726Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"0dc4bc9f-7044-47c0-86e1-70b77547b431","seconds":300}
{"level":"info","ts":"2026-11-03T15:07:04.731Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"fa4bfd7b-d314-42c5-b28e-9c5ef4c27f11","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:07:04.731Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"fa4bfd7b-d314-42c5-b28e-9c5ef4c27f11","name":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:07:04.731Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"fa4bfd7b-d314-42c5-b28e-9c5ef4c27f11","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:07:04.731Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"fa4bfd7b-d314-42c5-b28e-9c5ef4c27f11"}
{"level":"info","ts":"2026-11-03T15:07:04.731Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"fa4bfd7b-d314-42c5-b28e-9c5ef4c27f11","RayCluster":"stop-254kg","State":""}
{"level":"info","ts":"2026-11-03T15:07:04.731Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"fa4bfd7b-d314-42c5-b28e-9c5ef4c27f11","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:07:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:07:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:07:04.734Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"04b46b6c-99dc-4123-9719-654c134d7e86"}
{"level":"info","ts":"2026-11-03T15:07:04.734Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"04b46b6c-99dc-4123-9719-654c134d7e86"}
{"level":"info","ts":"2026-11-03T15:07:04.734Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"04b46b6c-99dc-4123-9719-654c134d7e86","1 head service found":"stop-254kg-head-svc"}
{"level":"info","ts":"2026-11-03T15:07:04.734Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"04b46b6c-99dc-4123-9719-654c134d7e86","Found 1 head Pod":"stop-254kg-head-4nd85","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:07:04.734Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"04b46b6c-99dc-4123-9719-654c134d7e86","head Pod":"stop-254kg-head-4nd85","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-254kg-head-4nd85. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:07:04.734Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"04b46b6c-99dc-4123-9719-654c134d7e86","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:07:04.734Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"04b46b6c-99dc-4123-9719-654c134d7e86","worker Pod":"stop-254kg-small-group-worker-nhg8h","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-254kg-small-group-worker-nhg8h. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:07:04.734Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"04b46b6c-99dc-4123-9719-654c134d7e86","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:07:04.734Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"04b46b6c-99dc-4123-9719-654c134d7e86","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:07:04.734Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"04b46b6c-99dc-4123-9719-654c134d7e86","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:07:04.734Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"04b46b6c-99dc-4123-9719-654c134d7e86","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:07:04.734Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"04b46b6c-99dc-4123-9719-654c134d7e86","podName":"stop-254kg-small-group-worker-nhg8h","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:07:04.734Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"04b46b6c-99dc-4123-9719-654c134d7e86","podName":"stop-254kg-small-group-worker-nhg8h","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:07:04.734Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"04b46b6c-99dc-4123-9719-654c134d7e86","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:07:04.734Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"04b46b6c-99dc-4123-9719-654c134d7e86","seconds":300}
{"level":"info","ts":"2026-11-03T15:07:05.197Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"62ec3d6b-b307-40ca-9563-ed317f0ccda0"}
{"level":"info","ts":"2026-11-03T15:07:05.197Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"62ec3d6b-b307-40ca-9563-ed317f0ccda0"}
{"level":"info","ts":"2026-11-03T15:07:05.197Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"62ec3d6b-b307-40ca-9563-ed317f0ccda0","1 head service found":"stop-254kg-head-svc"}
{"level":"info","ts":"2026-11-03T15:07:05.197Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"62ec3d6b-b307-40ca-9563-ed317f0ccda0","Found 1 head Pod":"stop-254kg-head-4nd85","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:07:05.197Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"62ec3d6b-b307-40ca-9563-ed317f0ccda0","head Pod":"stop-254kg-head-4nd85","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-254kg-head-4nd85. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:07:05.197Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"62ec3d6b-b307-40ca-9563-ed317f0ccda0","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:07:05.197Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"62ec3d6b-b307-40ca-9563-ed317f0ccda0","worker Pod":"stop-254kg-small-group-worker-nhg8h","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-254kg-small-group-worker-nhg8h. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:07:05.197Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"62ec3d6b-b307-40ca-9563-ed317f0ccda0","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:07:05.197Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"62ec3d6b-b307-40ca-9563-ed317f0ccda0","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:07:05.197Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"62ec3d6b-b307-40ca-9563-ed317f0ccda0","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:07:05.198Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"62ec3d6b-b307-40ca-9563-ed317f0ccda0","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:07:05.198Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"62ec3d6b-b307-40ca-9563-ed317f0ccda0","podName":"stop-254kg-head-4nd85","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:07:05.198Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"62ec3d6b-b307-40ca-9563-ed317f0ccda0","podName":"stop-254kg-head-4nd85","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:07:05.198Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"62ec3d6b-b307-40ca-9563-ed317f0ccda0","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:07:05.198Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"62ec3d6b-b307-40ca-9563-ed317f0ccda0","seconds":300}
{"level":"info","ts":"2026-11-03T15:07:05.716Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"5854089d-990c-4e24-a1dc-b1aa4ac38947"}
{"level":"info","ts":"2026-11-03T15:07:05.716Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"5854089d-990c-4e24-a1dc-b1aa4ac38947"}
{"level":"info","ts":"2026-11-03T15:07:05.716Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"5854089d-990c-4e24-a1dc-b1aa4ac38947","1 head service found":"stop-254kg-head-svc"}
{"level":"info","ts":"2026-11-03T15:07:05.717Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"5854089d-990c-4e24-a1dc-b1aa4ac38947","Found 1 head Pod":"stop-254kg-head-4nd85","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:07:05.717Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"5854089d-990c-4e24-a1dc-b1aa4ac38947","head Pod":"stop-254kg-head-4nd85","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-254kg-head-4nd85. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:07:05.717Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"5854089d-990c-4e24-a1dc-b1aa4ac38947","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:07:05.717Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"5854089d-990c-4e24-a1dc-b1aa4ac38947","worker Pod":"stop-254kg-small-group-worker-nhg8h","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-254kg-small-group-worker-nhg8h. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:07:05.717Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"5854089d-990c-4e24-a1dc-b1aa4ac38947","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:07:05.717Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"5854089d-990c-4e24-a1dc-b1aa4ac38947","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:07:05.717Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"5854089d-990c-4e24-a1dc-b1aa4ac38947","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:07:05.717Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"5854089d-990c-4e24-a1dc-b1aa4ac38947","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:07:05.717Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"5854089d-990c-4e24-a1dc-b1aa4ac38947","podName":"stop-254kg-head-4nd85","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:07:05.717Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"5854089d-990c-4e24-a1dc-b1aa4ac38947","podName":"stop-254kg-head-4nd85","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:07:05.717Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"5854089d-990c-4e24-a1dc-b1aa4ac38947","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:07:05.717Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"5854089d-990c-4e24-a1dc-b1aa4ac38947","seconds":300}
{"level":"info","ts":"2026-11-03T15:07:05.873Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"08533a8b-ad5e-48c2-9fa8-c57730769763"}
{"level":"info","ts":"2026-11-03T15:07:05.873Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"08533a8b-ad5e-48c2-9fa8-c57730769763"}
{"level":"info","ts":"2026-11-03T15:07:05.873Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"08533a8b-ad5e-48c2-9fa8-c57730769763","1 head service found":"stop-254kg-head-svc"}
{"level":"info","ts":"2026-11-03T15:07:05.873Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"08533a8b-ad5e-48c2-9fa8-c57730769763","Found 1 head Pod":"stop-254kg-head-4nd85","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:07:05.873Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"08533a8b-ad5e-48c2-9fa8-c57730769763","head Pod":"stop-254kg-head-4nd85","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-254kg-head-4nd85. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:07:05.873Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"08533a8b-ad5e-48c2-9fa8-c57730769763","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:07:05.873Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"08533a8b-ad5e-48c2-9fa8-c57730769763","worker Pod":"stop-254kg-small-group-worker-nhg8h","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-254kg-small-group-worker-nhg8h. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:07:05.873Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"08533a8b-ad5e-48c2-9fa8-c57730769763","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:07:05.873Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"08533a8b-ad5e-48c2-9fa8-c57730769763","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:07:05.873Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"08533a8b-ad5e-48c2-9fa8-c57730769763","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:07:05.873Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"08533a8b-ad5e-48c2-9fa8-c57730769763","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:07:05.873Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"08533a8b-ad5e-48c2-9fa8-c57730769763","podName":"stop-254kg-head-4nd85","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:07:05.873Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"08533a8b-ad5e-48c2-9fa8-c57730769763","podName":"stop-254kg-head-4nd85","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:07:05.873Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"08533a8b-ad5e-48c2-9fa8-c57730769763","name":"stop-254kg","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:07:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:07:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:07:05.934Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"da8db97b-2c69-4285-a522-a57158e37d7a","cluster":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:07:05.934Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"e0682b36-4ef0-43c6-adec-69736c84d206","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:07:05.934Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"e0682b36-4ef0-43c6-adec-69736c84d206","name":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:07:05.934Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"e0682b36-4ef0-43c6-adec-69736c84d206","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:07:05.935Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"e0682b36-4ef0-43c6-adec-69736c84d206"}
{"level":"info","ts":"2026-11-03T15:07:05.935Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"e0682b36-4ef0-43c6-adec-69736c84d206","RayCluster":"stop-254kg","State":""}
{"level":"info","ts":"2026-11-03T15:07:05.935Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"e0682b36-4ef0-43c6-adec-69736c84d206","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:07:04Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:07:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:07:05.935Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"e0682b36-4ef0-43c6-adec-69736c84d206","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:07:05.995Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"f8e9a94f-03aa-4fb3-980b-cec49c3c2303","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:07:05.995Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"f8e9a94f-03aa-4fb3-980b-cec49c3c2303","name":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:07:05.995Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"f8e9a94f-03aa-4fb3-980b-cec49c3c2303","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:07:05.995Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"f8e9a94f-03aa-4fb3-980b-cec49c3c2303"}
{"level":"info","ts":"2026-11-03T15:07:05.995Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"f8e9a94f-03aa-4fb3-980b-cec49c3c2303","RayCluster":"stop-254kg","State":""}
{"level":"info","ts":"2026-11-03T15:07:05.995Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"f8e9a94f-03aa-4fb3-980b-cec49c3c2303","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:07:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:07:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:07:06.241Z","logger":"controllers.RayClusterMTLS","msg":"Updating head certificate","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"da8db97b-2c69-4285-a522-a57158e37d7a","certificate":"ray-head-cert-stop-254kg","ipAddresses":["10.132.0.33","127.0.0.1"]}
{"level":"info","ts":"2026-11-03T15:07:06.650Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"da8db97b-2c69-4285-a522-a57158e37d7a","certificate":"ray-head-cert-stop-254kg"}
{"level":"info","ts":"2026-11-03T15:07:06.650Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"da8db97b-2c69-4285-a522-a57158e37d7a"}
{"level":"info","ts":"2026-11-03T15:07:06.697Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"acfaa2d9-6fb6-407d-b40a-62ff8ea3daed"}
{"level":"info","ts":"2026-11-03T15:07:06.698Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"acfaa2d9-6fb6-407d-b40a-62ff8ea3daed"}
{"level":"info","ts":"2026-11-03T15:07:06.698Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"acfaa2d9-6fb6-407d-b40a-62ff8ea3daed","1 head service found":"stop-254kg-head-svc"}
{"level":"info","ts":"2026-11-03T15:07:06.698Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"acfaa2d9-6fb6-407d-b40a-62ff8ea3daed","Found 1 head Pod":"stop-254kg-head-4nd85","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:07:06.698Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"acfaa2d9-6fb6-407d-b40a-62ff8ea3daed","head Pod":"stop-254kg-head-4nd85","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-254kg-head-4nd85. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:07:06.698Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"acfaa2d9-6fb6-407d-b40a-62ff8ea3daed","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:07:06.698Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"acfaa2d9-6fb6-407d-b40a-62ff8ea3daed","worker Pod":"stop-254kg-small-group-worker-nhg8h","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-254kg-small-group-worker-nhg8h. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:07:06.698Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"acfaa2d9-6fb6-407d-b40a-62ff8ea3daed","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:07:06.698Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"acfaa2d9-6fb6-407d-b40a-62ff8ea3daed","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:07:06.698Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"acfaa2d9-6fb6-407d-b40a-62ff8ea3daed","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:07:06.698Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"acfaa2d9-6fb6-407d-b40a-62ff8ea3daed","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:07:06.698Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"acfaa2d9-6fb6-407d-b40a-62ff8ea3daed","podName":"stop-254kg-head-4nd85","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:07:06.698Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"acfaa2d9-6fb6-407d-b40a-62ff8ea3daed","podName":"stop-254kg-head-4nd85","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:07:06.698Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"acfaa2d9-6fb6-407d-b40a-62ff8ea3daed","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:07:06.698Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"acfaa2d9-6fb6-407d-b40a-62ff8ea3daed","seconds":300}
{"level":"info","ts":"2026-11-03T15:07:08.996Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"8cc01841-c0d2-4399-b751-aea57ea547f3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:07:08.997Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"8cc01841-c0d2-4399-b751-aea57ea547f3","name":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:07:08.997Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"8cc01841-c0d2-4399-b751-aea57ea547f3","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:07:08.997Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"8cc01841-c0d2-4399-b751-aea57ea547f3"}
{"level":"info","ts":"2026-11-03T15:07:08.997Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"8cc01841-c0d2-4399-b751-aea57ea547f3","RayCluster":"stop-254kg","State":""}
{"level":"info","ts":"2026-11-03T15:07:08.997Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"8cc01841-c0d2-4399-b751-aea57ea547f3","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:07:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:07:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:07:11.998Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"d21a1af0-d224-4c75-a14b-390b0e517874","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:07:11.998Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"d21a1af0-d224-4c75-a14b-390b0e517874","name":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:07:11.998Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"d21a1af0-d224-4c75-a14b-390b0e517874","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:07:11.998Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"d21a1af0-d224-4c75-a14b-390b0e517874"}
{"level":"info","ts":"2026-11-03T15:07:11.998Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"d21a1af0-d224-4c75-a14b-390b0e517874","RayCluster":"stop-254kg","State":""}
{"level":"info","ts":"2026-11-03T15:07:11.998Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"d21a1af0-d224-4c75-a14b-390b0e517874","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:07:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:07:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:07:14.999Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"0b1d04f3-072b-4238-8764-8f7c990422bb","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:07:14.999Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"0b1d04f3-072b-4238-8764-8f7c990422bb","name":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:07:15.000Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"0b1d04f3-072b-4238-8764-8f7c990422bb","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:07:15.000Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"0b1d04f3-072b-4238-8764-8f7c990422bb"}
{"level":"info","ts":"2026-11-03T15:07:15.000Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"0b1d04f3-072b-4238-8764-8f7c990422bb","RayCluster":"stop-254kg","State":""}
{"level":"info","ts":"2026-11-03T15:07:15.000Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"0b1d04f3-072b-4238-8764-8f7c990422bb","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:07:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:07:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:07:18.000Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"1d7e2373-1bda-4a93-bd7e-5dc669e498ed","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:07:18.000Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"1d7e2373-1bda-4a93-bd7e-5dc669e498ed","name":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:07:18.001Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"1d7e2373-1bda-4a93-bd7e-5dc669e498ed","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:07:18.001Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"1d7e2373-1bda-4a93-bd7e-5dc669e498ed"}
{"level":"info","ts":"2026-11-03T15:07:18.001Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"1d7e2373-1bda-4a93-bd7e-5dc669e498ed","RayCluster":"stop-254kg","State":""}
{"level":"info","ts":"2026-11-03T15:07:18.001Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"1d7e2373-1bda-4a93-bd7e-5dc669e498ed","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:07:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:07:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:07:21.002Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"e60a6f0c-27d5-4c99-8a5d-82fe87de158e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:07:21.002Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"e60a6f0c-27d5-4c99-8a5d-82fe87de158e","name":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:07:21.002Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"e60a6f0c-27d5-4c99-8a5d-82fe87de158e","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:07:21.002Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"e60a6f0c-27d5-4c99-8a5d-82fe87de158e"}
{"level":"info","ts":"2026-11-03T15:07:21.002Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"e60a6f0c-27d5-4c99-8a5d-82fe87de158e","RayCluster":"stop-254kg","State":""}
{"level":"info","ts":"2026-11-03T15:07:21.002Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"e60a6f0c-27d5-4c99-8a5d-82fe87de158e","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:07:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:07:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:07:24.003Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"0e45a717-d644-419b-9e32-c29ebcf10d95","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:07:24.003Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"0e45a717-d644-419b-9e32-c29ebcf10d95","name":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:07:24.003Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"0e45a717-d644-419b-9e32-c29ebcf10d95","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:07:24.003Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"0e45a717-d644-419b-9e32-c29ebcf10d95"}
{"level":"info","ts":"2026-11-03T15:07:24.003Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"0e45a717-d644-419b-9e32-c29ebcf10d95","RayCluster":"stop-254kg","State":""}
{"level":"info","ts":"2026-11-03T15:07:24.003Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"0e45a717-d644-419b-9e32-c29ebcf10d95","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:07:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:07:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:07:27.004Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"b27fb1dc-24a3-41b5-9003-84c03351abd9","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:07:27.004Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"b27fb1dc-24a3-41b5-9003-84c03351abd9","name":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:07:27.004Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"b27fb1dc-24a3-41b5-9003-84c03351abd9","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:07:27.004Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"b27fb1dc-24a3-41b5-9003-84c03351abd9"}
{"level":"info","ts":"2026-11-03T15:07:27.004Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"b27fb1dc-24a3-41b5-9003-84c03351abd9","RayCluster":"stop-254kg","State":""}
{"level":"info","ts":"2026-11-03T15:07:27.004Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"b27fb1dc-24a3-41b5-9003-84c03351abd9","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:07:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:07:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:07:30.005Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"182386fc-9d8b-4dbf-ae8e-33db7c0d4140","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:07:30.005Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"182386fc-9d8b-4dbf-ae8e-33db7c0d4140","name":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:07:30.005Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"182386fc-9d8b-4dbf-ae8e-33db7c0d4140","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:07:30.005Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"182386fc-9d8b-4dbf-ae8e-33db7c0d4140"}
{"level":"info","ts":"2026-11-03T15:07:30.005Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"182386fc-9d8b-4dbf-ae8e-33db7c0d4140","RayCluster":"stop-254kg","State":""}
{"level":"info","ts":"2026-11-03T15:07:30.005Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"182386fc-9d8b-4dbf-ae8e-33db7c0d4140","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:07:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:07:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:07:33.006Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"0946d64d-0a3a-4da1-8472-798ec82992eb","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:07:33.006Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"0946d64d-0a3a-4da1-8472-798ec82992eb","name":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:07:33.006Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"0946d64d-0a3a-4da1-8472-798ec82992eb","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:07:33.007Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"0946d64d-0a3a-4da1-8472-798ec82992eb"}
{"level":"info","ts":"2026-11-03T15:07:33.007Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"0946d64d-0a3a-4da1-8472-798ec82992eb","RayCluster":"stop-254kg","State":""}
{"level":"info","ts":"2026-11-03T15:07:33.007Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"0946d64d-0a3a-4da1-8472-798ec82992eb","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:07:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:07:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:07:36.008Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"f0909fbd-d302-4d62-ae35-de79ed0fc353","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:07:36.008Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"f0909fbd-d302-4d62-ae35-de79ed0fc353","name":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:07:36.008Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"f0909fbd-d302-4d62-ae35-de79ed0fc353","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:07:36.008Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"f0909fbd-d302-4d62-ae35-de79ed0fc353"}
{"level":"info","ts":"2026-11-03T15:07:36.008Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"f0909fbd-d302-4d62-ae35-de79ed0fc353","RayCluster":"stop-254kg","State":""}
{"level":"info","ts":"2026-11-03T15:07:36.008Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"f0909fbd-d302-4d62-ae35-de79ed0fc353","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:07:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:07:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:07:36.651Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"ec5587e9-d492-4a11-8d34-84717082f050","cluster":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:07:37.105Z","logger":"controllers.RayClusterMTLS","msg":"Updating worker certificate","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"ec5587e9-d492-4a11-8d34-84717082f050","certificate":"ray-worker-cert-stop-254kg","ipAddresses":["10.134.0.29","127.0.0.1"]}
{"level":"info","ts":"2026-11-03T15:07:37.419Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"ec5587e9-d492-4a11-8d34-84717082f050","certificate":"ray-worker-cert-stop-254kg"}
{"level":"info","ts":"2026-11-03T15:07:37.419Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"ec5587e9-d492-4a11-8d34-84717082f050"}
{"level":"info","ts":"2026-11-03T15:07:39.009Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"b4500e24-7eba-44eb-8129-3545a016a319","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:07:39.009Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"b4500e24-7eba-44eb-8129-3545a016a319","name":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:07:39.009Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"b4500e24-7eba-44eb-8129-3545a016a319","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:07:39.009Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"b4500e24-7eba-44eb-8129-3545a016a319"}
{"level":"info","ts":"2026-11-03T15:07:39.009Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"b4500e24-7eba-44eb-8129-3545a016a319","RayCluster":"stop-254kg","State":""}
{"level":"info","ts":"2026-11-03T15:07:39.009Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"b4500e24-7eba-44eb-8129-3545a016a319","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:07:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:07:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:07:42.010Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"5ea9ddf6-e441-4485-bfdd-b17f3e0ca6e4","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:07:42.010Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"5ea9ddf6-e441-4485-bfdd-b17f3e0ca6e4","name":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:07:42.010Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"5ea9ddf6-e441-4485-bfdd-b17f3e0ca6e4","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:07:42.010Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"5ea9ddf6-e441-4485-bfdd-b17f3e0ca6e4"}
{"level":"info","ts":"2026-11-03T15:07:42.010Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"5ea9ddf6-e441-4485-bfdd-b17f3e0ca6e4","RayCluster":"stop-254kg","State":""}
{"level":"info","ts":"2026-11-03T15:07:42.010Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"5ea9ddf6-e441-4485-bfdd-b17f3e0ca6e4","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:07:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:07:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:07:45.012Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"3917e557-d05e-42a5-9bb8-918ceaec9b8b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:07:45.012Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"3917e557-d05e-42a5-9bb8-918ceaec9b8b","name":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:07:45.012Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"3917e557-d05e-42a5-9bb8-918ceaec9b8b","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:07:45.012Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"3917e557-d05e-42a5-9bb8-918ceaec9b8b"}
{"level":"info","ts":"2026-11-03T15:07:45.012Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"3917e557-d05e-42a5-9bb8-918ceaec9b8b","RayCluster":"stop-254kg","State":""}
{"level":"info","ts":"2026-11-03T15:07:45.012Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"3917e557-d05e-42a5-9bb8-918ceaec9b8b","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:07:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:07:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:07:48.013Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"c9ee4456-1ba9-4766-8fd7-4d1dde6c72ce","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:07:48.013Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"c9ee4456-1ba9-4766-8fd7-4d1dde6c72ce","name":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:07:48.013Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"c9ee4456-1ba9-4766-8fd7-4d1dde6c72ce","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:07:48.013Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"c9ee4456-1ba9-4766-8fd7-4d1dde6c72ce"}
{"level":"info","ts":"2026-11-03T15:07:48.013Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"c9ee4456-1ba9-4766-8fd7-4d1dde6c72ce","RayCluster":"stop-254kg","State":""}
{"level":"info","ts":"2026-11-03T15:07:48.013Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"c9ee4456-1ba9-4766-8fd7-4d1dde6c72ce","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:07:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:07:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:07:51.014Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"7614c09e-95ea-4df8-8c62-cbf4eeb526e0","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:07:51.014Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"7614c09e-95ea-4df8-8c62-cbf4eeb526e0","name":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:07:51.014Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"7614c09e-95ea-4df8-8c62-cbf4eeb526e0","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:07:51.015Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"7614c09e-95ea-4df8-8c62-cbf4eeb526e0"}
{"level":"info","ts":"2026-11-03T15:07:51.015Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"7614c09e-95ea-4df8-8c62-cbf4eeb526e0","RayCluster":"stop-254kg","State":""}
{"level":"info","ts":"2026-11-03T15:07:51.015Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"7614c09e-95ea-4df8-8c62-cbf4eeb526e0","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:07:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:07:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:07:54.016Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"455dd942-4229-4fcd-a442-d80bf59452c1","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:07:54.016Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"455dd942-4229-4fcd-a442-d80bf59452c1","name":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:07:54.016Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"455dd942-4229-4fcd-a442-d80bf59452c1","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:07:54.016Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"455dd942-4229-4fcd-a442-d80bf59452c1"}
{"level":"info","ts":"2026-11-03T15:07:54.016Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"455dd942-4229-4fcd-a442-d80bf59452c1","RayCluster":"stop-254kg","State":""}
{"level":"info","ts":"2026-11-03T15:07:54.016Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"455dd942-4229-4fcd-a442-d80bf59452c1","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:07:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:07:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:07:57.017Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"348f293c-2c3d-4b7b-98cf-a06be2c6a10b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:07:57.017Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"348f293c-2c3d-4b7b-98cf-a06be2c6a10b","name":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:07:57.017Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"348f293c-2c3d-4b7b-98cf-a06be2c6a10b","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:07:57.017Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"348f293c-2c3d-4b7b-98cf-a06be2c6a10b"}
{"level":"info","ts":"2026-11-03T15:07:57.017Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"348f293c-2c3d-4b7b-98cf-a06be2c6a10b","RayCluster":"stop-254kg","State":""}
{"level":"info","ts":"2026-11-03T15:07:57.017Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"348f293c-2c3d-4b7b-98cf-a06be2c6a10b","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:07:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:07:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:08:00.018Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"67e88f36-e957-4f41-99f7-6e795601cf5e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:08:00.018Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"67e88f36-e957-4f41-99f7-6e795601cf5e","name":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:08:00.018Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"67e88f36-e957-4f41-99f7-6e795601cf5e","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:08:00.018Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"67e88f36-e957-4f41-99f7-6e795601cf5e"}
{"level":"info","ts":"2026-11-03T15:08:00.018Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"67e88f36-e957-4f41-99f7-6e795601cf5e","RayCluster":"stop-254kg","State":""}
{"level":"info","ts":"2026-11-03T15:08:00.018Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"67e88f36-e957-4f41-99f7-6e795601cf5e","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:07:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:07:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:08:03.019Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"7567d737-50d3-4d30-afd7-ee6e73f5a9c1","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:08:03.019Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"7567d737-50d3-4d30-afd7-ee6e73f5a9c1","name":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:08:03.019Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"7567d737-50d3-4d30-afd7-ee6e73f5a9c1","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:08:03.019Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"7567d737-50d3-4d30-afd7-ee6e73f5a9c1"}
{"level":"info","ts":"2026-11-03T15:08:03.019Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"7567d737-50d3-4d30-afd7-ee6e73f5a9c1","RayCluster":"stop-254kg","State":""}
{"level":"info","ts":"2026-11-03T15:08:03.019Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"7567d737-50d3-4d30-afd7-ee6e73f5a9c1","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:07:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:07:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:08:06.021Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"ce9b18de-2a78-40ea-b2df-f8c4c1472cad","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:08:06.021Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"ce9b18de-2a78-40ea-b2df-f8c4c1472cad","name":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:08:06.021Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"ce9b18de-2a78-40ea-b2df-f8c4c1472cad","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:08:06.021Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"ce9b18de-2a78-40ea-b2df-f8c4c1472cad"}
{"level":"info","ts":"2026-11-03T15:08:06.021Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"ce9b18de-2a78-40ea-b2df-f8c4c1472cad","RayCluster":"stop-254kg","State":""}
{"level":"info","ts":"2026-11-03T15:08:06.021Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"ce9b18de-2a78-40ea-b2df-f8c4c1472cad","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:07:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:07:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:08:07.420Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"adc8f838-0b3c-44e5-9058-e2028b9db087","cluster":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:08:09.022Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"a4497877-4526-4758-869f-d24ef952aa3e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:08:09.022Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"a4497877-4526-4758-869f-d24ef952aa3e","name":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:08:09.022Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"a4497877-4526-4758-869f-d24ef952aa3e","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:08:09.022Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"a4497877-4526-4758-869f-d24ef952aa3e"}
{"level":"info","ts":"2026-11-03T15:08:09.022Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"a4497877-4526-4758-869f-d24ef952aa3e","RayCluster":"stop-254kg","State":""}
{"level":"info","ts":"2026-11-03T15:08:09.022Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"a4497877-4526-4758-869f-d24ef952aa3e","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:07:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:07:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:08:12.023Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"199ff8eb-4a85-445c-a283-52a3223bae18","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:08:12.023Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"199ff8eb-4a85-445c-a283-52a3223bae18","name":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:08:12.023Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"199ff8eb-4a85-445c-a283-52a3223bae18","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:08:12.023Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"199ff8eb-4a85-445c-a283-52a3223bae18"}
{"level":"info","ts":"2026-11-03T15:08:12.023Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"199ff8eb-4a85-445c-a283-52a3223bae18","RayCluster":"stop-254kg","State":""}
{"level":"info","ts":"2026-11-03T15:08:12.023Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"199ff8eb-4a85-445c-a283-52a3223bae18","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:07:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:07:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:08:15.024Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"350229c9-be64-4625-8432-0a3be9354186","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:08:15.025Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"350229c9-be64-4625-8432-0a3be9354186","name":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:08:15.025Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"350229c9-be64-4625-8432-0a3be9354186","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:08:15.025Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"350229c9-be64-4625-8432-0a3be9354186"}
{"level":"info","ts":"2026-11-03T15:08:15.025Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"350229c9-be64-4625-8432-0a3be9354186","RayCluster":"stop-254kg","State":""}
{"level":"info","ts":"2026-11-03T15:08:15.025Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"350229c9-be64-4625-8432-0a3be9354186","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:07:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:07:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:08:15.227Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"8f24cc96-9a00-4cb6-876e-f9d5a8f28850"}
{"level":"info","ts":"2026-11-03T15:08:15.227Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"8f24cc96-9a00-4cb6-876e-f9d5a8f28850"}
{"level":"info","ts":"2026-11-03T15:08:15.227Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"8f24cc96-9a00-4cb6-876e-f9d5a8f28850","1 head service found":"stop-254kg-head-svc"}
{"level":"info","ts":"2026-11-03T15:08:15.227Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"8f24cc96-9a00-4cb6-876e-f9d5a8f28850","Found 1 head Pod":"stop-254kg-head-4nd85","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:08:15.227Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"8f24cc96-9a00-4cb6-876e-f9d5a8f28850","head Pod":"stop-254kg-head-4nd85","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-254kg-head-4nd85. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:08:15.227Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"8f24cc96-9a00-4cb6-876e-f9d5a8f28850","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:08:15.227Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"8f24cc96-9a00-4cb6-876e-f9d5a8f28850","worker Pod":"stop-254kg-small-group-worker-nhg8h","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-254kg-small-group-worker-nhg8h. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:08:15.227Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"8f24cc96-9a00-4cb6-876e-f9d5a8f28850","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:08:15.227Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"8f24cc96-9a00-4cb6-876e-f9d5a8f28850","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:08:15.227Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"8f24cc96-9a00-4cb6-876e-f9d5a8f28850","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:08:15.227Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"8f24cc96-9a00-4cb6-876e-f9d5a8f28850","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:08:15.227Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"8f24cc96-9a00-4cb6-876e-f9d5a8f28850","podName":"stop-254kg-head-4nd85","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:08:15.227Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"8f24cc96-9a00-4cb6-876e-f9d5a8f28850","podName":"stop-254kg-head-4nd85","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:08:15.227Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"8f24cc96-9a00-4cb6-876e-f9d5a8f28850","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:08:15.227Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"8f24cc96-9a00-4cb6-876e-f9d5a8f28850","seconds":300}
{"level":"info","ts":"2026-11-03T15:08:16.219Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"b35f3c94-e049-4b7a-8e7f-14966ec3b02b"}
{"level":"info","ts":"2026-11-03T15:08:16.219Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"b35f3c94-e049-4b7a-8e7f-14966ec3b02b"}
{"level":"info","ts":"2026-11-03T15:08:16.220Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"b35f3c94-e049-4b7a-8e7f-14966ec3b02b","1 head service found":"stop-254kg-head-svc"}
{"level":"info","ts":"2026-11-03T15:08:16.220Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"b35f3c94-e049-4b7a-8e7f-14966ec3b02b","Found 1 head Pod":"stop-254kg-head-4nd85","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:08:16.220Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"b35f3c94-e049-4b7a-8e7f-14966ec3b02b","head Pod":"stop-254kg-head-4nd85","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-254kg-head-4nd85. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:08:16.220Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"b35f3c94-e049-4b7a-8e7f-14966ec3b02b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:08:16.220Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"b35f3c94-e049-4b7a-8e7f-14966ec3b02b","worker Pod":"stop-254kg-small-group-worker-nhg8h","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-254kg-small-group-worker-nhg8h. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:08:16.220Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"b35f3c94-e049-4b7a-8e7f-14966ec3b02b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:08:16.220Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"b35f3c94-e049-4b7a-8e7f-14966ec3b02b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:08:16.220Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"b35f3c94-e049-4b7a-8e7f-14966ec3b02b","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:08:16.220Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"b35f3c94-e049-4b7a-8e7f-14966ec3b02b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:08:16.220Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"b35f3c94-e049-4b7a-8e7f-14966ec3b02b","podName":"stop-254kg-head-4nd85","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:07:04Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:08:16.220Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"b35f3c94-e049-4b7a-8e7f-14966ec3b02b","podName":"stop-254kg-head-4nd85","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:07:04Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:08:16.220Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"b35f3c94-e049-4b7a-8e7f-14966ec3b02b","name":"stop-254kg","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:07:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:08:16Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:08:16.281Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"8e170f4b-3f32-4602-87e5-5d515c1835f4","cluster":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:08:16.282Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"1e069d50-eceb-449f-bd98-f029bb4f71e8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:08:16.282Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"1e069d50-eceb-449f-bd98-f029bb4f71e8","name":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:08:16.282Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"1e069d50-eceb-449f-bd98-f029bb4f71e8","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:08:16.282Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"1e069d50-eceb-449f-bd98-f029bb4f71e8"}
{"level":"info","ts":"2026-11-03T15:08:16.282Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"1e069d50-eceb-449f-bd98-f029bb4f71e8","RayCluster":"stop-254kg","State":""}
{"level":"info","ts":"2026-11-03T15:08:16.282Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"1e069d50-eceb-449f-bd98-f029bb4f71e8","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:07:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:08:16Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:08:16.282Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"1e069d50-eceb-449f-bd98-f029bb4f71e8","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:08:16.342Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"4b8948c1-49db-4a8e-b359-5c3d0a274ddc","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:08:16.342Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"4b8948c1-49db-4a8e-b359-5c3d0a274ddc","name":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:08:16.342Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"4b8948c1-49db-4a8e-b359-5c3d0a274ddc","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:08:16.343Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"4b8948c1-49db-4a8e-b359-5c3d0a274ddc"}
{"level":"info","ts":"2026-11-03T15:08:16.343Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"4b8948c1-49db-4a8e-b359-5c3d0a274ddc","RayCluster":"stop-254kg","State":""}
{"level":"info","ts":"2026-11-03T15:08:16.343Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"4b8948c1-49db-4a8e-b359-5c3d0a274ddc","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:08:16Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:08:16Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:08:18.281Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"c3d47449-0c7a-49c8-8c49-10fb44d4506c"}
{"level":"info","ts":"2026-11-03T15:08:18.282Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"c3d47449-0c7a-49c8-8c49-10fb44d4506c"}
{"level":"info","ts":"2026-11-03T15:08:18.282Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"c3d47449-0c7a-49c8-8c49-10fb44d4506c","1 head service found":"stop-254kg-head-svc"}
{"level":"info","ts":"2026-11-03T15:08:18.282Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"c3d47449-0c7a-49c8-8c49-10fb44d4506c","Found 1 head Pod":"stop-254kg-head-4nd85","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:08:18.282Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"c3d47449-0c7a-49c8-8c49-10fb44d4506c","head Pod":"stop-254kg-head-4nd85","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-254kg-head-4nd85. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:08:18.282Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"c3d47449-0c7a-49c8-8c49-10fb44d4506c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:08:18.282Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"c3d47449-0c7a-49c8-8c49-10fb44d4506c","worker Pod":"stop-254kg-small-group-worker-nhg8h","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-254kg-small-group-worker-nhg8h. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:08:18.282Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"c3d47449-0c7a-49c8-8c49-10fb44d4506c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:08:18.282Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"c3d47449-0c7a-49c8-8c49-10fb44d4506c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:08:18.282Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"c3d47449-0c7a-49c8-8c49-10fb44d4506c","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:08:18.282Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"c3d47449-0c7a-49c8-8c49-10fb44d4506c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:08:18.282Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"c3d47449-0c7a-49c8-8c49-10fb44d4506c","podName":"stop-254kg-head-4nd85","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:07:04Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:08:18.282Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"c3d47449-0c7a-49c8-8c49-10fb44d4506c","podName":"stop-254kg-head-4nd85","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:07:04Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:08:18.282Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"c3d47449-0c7a-49c8-8c49-10fb44d4506c","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:08:18.282Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"c3d47449-0c7a-49c8-8c49-10fb44d4506c","seconds":300}
{"level":"info","ts":"2026-11-03T15:08:19.343Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"7c61e276-80fa-41d8-a7c9-40fa4b72fba0","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:08:19.343Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"7c61e276-80fa-41d8-a7c9-40fa4b72fba0","name":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:08:19.343Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"7c61e276-80fa-41d8-a7c9-40fa4b72fba0","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:08:19.343Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"7c61e276-80fa-41d8-a7c9-40fa4b72fba0"}
{"level":"info","ts":"2026-11-03T15:08:19.343Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"7c61e276-80fa-41d8-a7c9-40fa4b72fba0","RayCluster":"stop-254kg","State":""}
{"level":"info","ts":"2026-11-03T15:08:19.343Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"7c61e276-80fa-41d8-a7c9-40fa4b72fba0","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:08:16Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:08:16Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:08:22.017Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"39a8b461-7681-4cb2-8582-5bd06cc5f93f"}
{"level":"info","ts":"2026-11-03T15:08:22.017Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"39a8b461-7681-4cb2-8582-5bd06cc5f93f"}
{"level":"info","ts":"2026-11-03T15:08:22.017Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"39a8b461-7681-4cb2-8582-5bd06cc5f93f","1 head service found":"stop-254kg-head-svc"}
{"level":"info","ts":"2026-11-03T15:08:22.017Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"39a8b461-7681-4cb2-8582-5bd06cc5f93f","Found 1 head Pod":"stop-254kg-head-4nd85","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:08:22.017Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"39a8b461-7681-4cb2-8582-5bd06cc5f93f","head Pod":"stop-254kg-head-4nd85","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-254kg-head-4nd85. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:08:22.017Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"39a8b461-7681-4cb2-8582-5bd06cc5f93f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:08:22.017Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"39a8b461-7681-4cb2-8582-5bd06cc5f93f","worker Pod":"stop-254kg-small-group-worker-nhg8h","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-254kg-small-group-worker-nhg8h. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:08:22.017Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"39a8b461-7681-4cb2-8582-5bd06cc5f93f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:08:22.017Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"39a8b461-7681-4cb2-8582-5bd06cc5f93f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:08:22.017Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"39a8b461-7681-4cb2-8582-5bd06cc5f93f","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:08:22.017Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"39a8b461-7681-4cb2-8582-5bd06cc5f93f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:08:22.017Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"39a8b461-7681-4cb2-8582-5bd06cc5f93f","podName":"stop-254kg-head-4nd85","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:07:04Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:08:22.017Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"39a8b461-7681-4cb2-8582-5bd06cc5f93f","podName":"stop-254kg-head-4nd85","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:07:04Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:08:22.017Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"39a8b461-7681-4cb2-8582-5bd06cc5f93f","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:08:22.017Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"39a8b461-7681-4cb2-8582-5bd06cc5f93f","seconds":300}
{"level":"info","ts":"2026-11-03T15:08:22.344Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"64da852e-954e-49fd-a073-a6ba0b848cfc","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:08:22.344Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"64da852e-954e-49fd-a073-a6ba0b848cfc","name":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:08:22.344Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"64da852e-954e-49fd-a073-a6ba0b848cfc","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:08:22.344Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"64da852e-954e-49fd-a073-a6ba0b848cfc"}
{"level":"info","ts":"2026-11-03T15:08:22.344Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"64da852e-954e-49fd-a073-a6ba0b848cfc","RayCluster":"stop-254kg","State":""}
{"level":"info","ts":"2026-11-03T15:08:22.344Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"64da852e-954e-49fd-a073-a6ba0b848cfc","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:08:16Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:08:16Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:08:23.021Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"5db51557-8830-452b-85ff-7acbec748ddc"}
{"level":"info","ts":"2026-11-03T15:08:23.021Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"5db51557-8830-452b-85ff-7acbec748ddc"}
{"level":"info","ts":"2026-11-03T15:08:23.021Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"5db51557-8830-452b-85ff-7acbec748ddc","1 head service found":"stop-254kg-head-svc"}
{"level":"info","ts":"2026-11-03T15:08:23.021Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"5db51557-8830-452b-85ff-7acbec748ddc","Found 1 head Pod":"stop-254kg-head-4nd85","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:08:23.021Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"5db51557-8830-452b-85ff-7acbec748ddc","head Pod":"stop-254kg-head-4nd85","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-254kg-head-4nd85. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:08:23.021Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"5db51557-8830-452b-85ff-7acbec748ddc","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:08:23.021Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"5db51557-8830-452b-85ff-7acbec748ddc","worker Pod":"stop-254kg-small-group-worker-nhg8h","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-254kg-small-group-worker-nhg8h. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:08:23.021Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"5db51557-8830-452b-85ff-7acbec748ddc","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:08:23.021Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"5db51557-8830-452b-85ff-7acbec748ddc","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:08:23.021Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"5db51557-8830-452b-85ff-7acbec748ddc","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:08:23.021Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"5db51557-8830-452b-85ff-7acbec748ddc","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:08:23.021Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"5db51557-8830-452b-85ff-7acbec748ddc","podName":"stop-254kg-head-4nd85","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:07:04Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:08:23.021Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"5db51557-8830-452b-85ff-7acbec748ddc","podName":"stop-254kg-head-4nd85","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:07:04Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:08:23.022Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"5db51557-8830-452b-85ff-7acbec748ddc","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:08:23.022Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"5db51557-8830-452b-85ff-7acbec748ddc","seconds":300}
{"level":"info","ts":"2026-11-03T15:08:25.345Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"f1c369e0-798b-4907-a81e-0b31e6c6a506","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:08:25.345Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"f1c369e0-798b-4907-a81e-0b31e6c6a506","name":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:08:25.345Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"f1c369e0-798b-4907-a81e-0b31e6c6a506","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:08:25.345Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"f1c369e0-798b-4907-a81e-0b31e6c6a506"}
{"level":"info","ts":"2026-11-03T15:08:25.345Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"f1c369e0-798b-4907-a81e-0b31e6c6a506","RayCluster":"stop-254kg","State":""}
{"level":"info","ts":"2026-11-03T15:08:25.345Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"f1c369e0-798b-4907-a81e-0b31e6c6a506","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:08:16Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:08:16Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:08:27.275Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"f65c8c62-0570-48c2-9fb2-2aeb8cf5bf71"}
{"level":"info","ts":"2026-11-03T15:08:27.275Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"f65c8c62-0570-48c2-9fb2-2aeb8cf5bf71"}
{"level":"info","ts":"2026-11-03T15:08:27.275Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"f65c8c62-0570-48c2-9fb2-2aeb8cf5bf71","1 head service found":"stop-254kg-head-svc"}
{"level":"info","ts":"2026-11-03T15:08:27.275Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"f65c8c62-0570-48c2-9fb2-2aeb8cf5bf71","Found 1 head Pod":"stop-254kg-head-4nd85","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:08:27.275Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"f65c8c62-0570-48c2-9fb2-2aeb8cf5bf71","head Pod":"stop-254kg-head-4nd85","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-254kg-head-4nd85. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:08:27.275Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"f65c8c62-0570-48c2-9fb2-2aeb8cf5bf71","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:08:27.275Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"f65c8c62-0570-48c2-9fb2-2aeb8cf5bf71","worker Pod":"stop-254kg-small-group-worker-nhg8h","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-254kg-small-group-worker-nhg8h. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:08:27.275Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"f65c8c62-0570-48c2-9fb2-2aeb8cf5bf71","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:08:27.275Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"f65c8c62-0570-48c2-9fb2-2aeb8cf5bf71","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:08:27.275Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"f65c8c62-0570-48c2-9fb2-2aeb8cf5bf71","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:08:27.275Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"f65c8c62-0570-48c2-9fb2-2aeb8cf5bf71","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:08:27.275Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"f65c8c62-0570-48c2-9fb2-2aeb8cf5bf71","podName":"stop-254kg-small-group-worker-nhg8h","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:08:27.275Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"f65c8c62-0570-48c2-9fb2-2aeb8cf5bf71","podName":"stop-254kg-small-group-worker-nhg8h","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:08:27.275Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"f65c8c62-0570-48c2-9fb2-2aeb8cf5bf71","name":"stop-254kg","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:08:16Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:08:27Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:08:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:08:27.336Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"61c750c1-3bf7-4078-a709-51b22447b60f","cluster":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:08:27.336Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"409d0c09-a065-4b5a-8563-282f4addb346","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:08:27.336Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"409d0c09-a065-4b5a-8563-282f4addb346","name":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:08:27.336Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"409d0c09-a065-4b5a-8563-282f4addb346","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:08:27.336Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"409d0c09-a065-4b5a-8563-282f4addb346"}
{"level":"info","ts":"2026-11-03T15:08:27.336Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"409d0c09-a065-4b5a-8563-282f4addb346","RayCluster":"stop-254kg","State":""}
{"level":"info","ts":"2026-11-03T15:08:27.336Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"409d0c09-a065-4b5a-8563-282f4addb346","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:08:16Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:08:27Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:08:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:08:27.336Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"409d0c09-a065-4b5a-8563-282f4addb346","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:08:27.397Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"d8eba813-59d8-4b1b-90c8-68b69cfaba31","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:08:27.397Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"d8eba813-59d8-4b1b-90c8-68b69cfaba31","name":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:08:27.397Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"d8eba813-59d8-4b1b-90c8-68b69cfaba31","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:08:27.397Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"d8eba813-59d8-4b1b-90c8-68b69cfaba31"}
{"level":"info","ts":"2026-11-03T15:08:27.397Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"d8eba813-59d8-4b1b-90c8-68b69cfaba31","RayCluster":"stop-254kg","State":""}
{"level":"info","ts":"2026-11-03T15:08:27.397Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"d8eba813-59d8-4b1b-90c8-68b69cfaba31","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:08:27Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:08:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:08:27Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:08:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:08:29.336Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"10001bbe-2d28-438e-87dc-786a2e7a9c21"}
{"level":"info","ts":"2026-11-03T15:08:29.336Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"10001bbe-2d28-438e-87dc-786a2e7a9c21"}
{"level":"info","ts":"2026-11-03T15:08:29.336Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"10001bbe-2d28-438e-87dc-786a2e7a9c21","1 head service found":"stop-254kg-head-svc"}
{"level":"info","ts":"2026-11-03T15:08:29.336Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"10001bbe-2d28-438e-87dc-786a2e7a9c21","Found 1 head Pod":"stop-254kg-head-4nd85","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:08:29.336Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"10001bbe-2d28-438e-87dc-786a2e7a9c21","head Pod":"stop-254kg-head-4nd85","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-254kg-head-4nd85. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:08:29.336Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"10001bbe-2d28-438e-87dc-786a2e7a9c21","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:08:29.336Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"10001bbe-2d28-438e-87dc-786a2e7a9c21","worker Pod":"stop-254kg-small-group-worker-nhg8h","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-254kg-small-group-worker-nhg8h. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:08:29.336Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"10001bbe-2d28-438e-87dc-786a2e7a9c21","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:08:29.336Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"10001bbe-2d28-438e-87dc-786a2e7a9c21","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:08:29.336Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"10001bbe-2d28-438e-87dc-786a2e7a9c21","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:08:29.336Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"10001bbe-2d28-438e-87dc-786a2e7a9c21","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:08:29.337Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"10001bbe-2d28-438e-87dc-786a2e7a9c21","podName":"stop-254kg-small-group-worker-nhg8h","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:08:29.337Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"10001bbe-2d28-438e-87dc-786a2e7a9c21","podName":"stop-254kg-small-group-worker-nhg8h","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:08:29.337Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"10001bbe-2d28-438e-87dc-786a2e7a9c21","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:08:29.337Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"10001bbe-2d28-438e-87dc-786a2e7a9c21","seconds":300}
{"level":"info","ts":"2026-11-03T15:08:30.397Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"d767d4cd-f63a-4fc6-8e08-b13f4e69c5a6","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:08:30.397Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"d767d4cd-f63a-4fc6-8e08-b13f4e69c5a6","name":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:08:30.397Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"d767d4cd-f63a-4fc6-8e08-b13f4e69c5a6","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:08:30.397Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"d767d4cd-f63a-4fc6-8e08-b13f4e69c5a6"}
{"level":"info","ts":"2026-11-03T15:08:30.398Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"d767d4cd-f63a-4fc6-8e08-b13f4e69c5a6","RayCluster":"stop-254kg","State":""}
{"level":"info","ts":"2026-11-03T15:08:30.398Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"d767d4cd-f63a-4fc6-8e08-b13f4e69c5a6","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:08:27Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:08:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:08:27Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:08:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:08:32.063Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"642f8850-6216-412a-a9d2-8b812e28562d"}
{"level":"info","ts":"2026-11-03T15:08:32.063Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"642f8850-6216-412a-a9d2-8b812e28562d"}
{"level":"info","ts":"2026-11-03T15:08:32.063Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"642f8850-6216-412a-a9d2-8b812e28562d","1 head service found":"stop-254kg-head-svc"}
{"level":"info","ts":"2026-11-03T15:08:32.063Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"642f8850-6216-412a-a9d2-8b812e28562d","Found 1 head Pod":"stop-254kg-head-4nd85","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:08:32.063Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"642f8850-6216-412a-a9d2-8b812e28562d","head Pod":"stop-254kg-head-4nd85","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-254kg-head-4nd85. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:08:32.063Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"642f8850-6216-412a-a9d2-8b812e28562d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:08:32.063Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"642f8850-6216-412a-a9d2-8b812e28562d","worker Pod":"stop-254kg-small-group-worker-nhg8h","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-254kg-small-group-worker-nhg8h. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:08:32.063Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"642f8850-6216-412a-a9d2-8b812e28562d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:08:32.063Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"642f8850-6216-412a-a9d2-8b812e28562d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:08:32.063Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"642f8850-6216-412a-a9d2-8b812e28562d","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:08:32.063Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"642f8850-6216-412a-a9d2-8b812e28562d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:08:32.063Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"642f8850-6216-412a-a9d2-8b812e28562d","podName":"stop-254kg-small-group-worker-nhg8h","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:08:32.063Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"642f8850-6216-412a-a9d2-8b812e28562d","podName":"stop-254kg-small-group-worker-nhg8h","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:08:32.063Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"642f8850-6216-412a-a9d2-8b812e28562d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:08:32.063Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"642f8850-6216-412a-a9d2-8b812e28562d","seconds":300}
{"level":"info","ts":"2026-11-03T15:08:33.070Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"f76b60da-bcc7-483e-82cd-33f082cc0620"}
{"level":"info","ts":"2026-11-03T15:08:33.070Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"f76b60da-bcc7-483e-82cd-33f082cc0620"}
{"level":"info","ts":"2026-11-03T15:08:33.070Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"f76b60da-bcc7-483e-82cd-33f082cc0620","1 head service found":"stop-254kg-head-svc"}
{"level":"info","ts":"2026-11-03T15:08:33.070Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"f76b60da-bcc7-483e-82cd-33f082cc0620","Found 1 head Pod":"stop-254kg-head-4nd85","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:08:33.070Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"f76b60da-bcc7-483e-82cd-33f082cc0620","head Pod":"stop-254kg-head-4nd85","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-254kg-head-4nd85. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:08:33.070Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"f76b60da-bcc7-483e-82cd-33f082cc0620","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:08:33.070Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"f76b60da-bcc7-483e-82cd-33f082cc0620","worker Pod":"stop-254kg-small-group-worker-nhg8h","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-254kg-small-group-worker-nhg8h. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:08:33.070Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"f76b60da-bcc7-483e-82cd-33f082cc0620","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:08:33.070Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"f76b60da-bcc7-483e-82cd-33f082cc0620","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:08:33.070Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"f76b60da-bcc7-483e-82cd-33f082cc0620","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:08:33.070Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"f76b60da-bcc7-483e-82cd-33f082cc0620","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:08:33.070Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"f76b60da-bcc7-483e-82cd-33f082cc0620","podName":"stop-254kg-small-group-worker-nhg8h","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:07:04Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:08:33.070Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"f76b60da-bcc7-483e-82cd-33f082cc0620","podName":"stop-254kg-small-group-worker-nhg8h","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:07:04Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:08:33.070Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"f76b60da-bcc7-483e-82cd-33f082cc0620","name":"stop-254kg","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:08:27Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:08:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:08:33Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:08:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:08:33.131Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"9111ac39-940a-4ffb-a7ef-53cc5919117a","cluster":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:08:33.131Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"b85330bd-cae8-44c1-ab6c-edb0e41155cb","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:08:33.131Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"b85330bd-cae8-44c1-ab6c-edb0e41155cb","name":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:08:33.131Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"b85330bd-cae8-44c1-ab6c-edb0e41155cb","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:08:33.132Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"b85330bd-cae8-44c1-ab6c-edb0e41155cb"}
{"level":"info","ts":"2026-11-03T15:08:33.132Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"b85330bd-cae8-44c1-ab6c-edb0e41155cb","RayCluster":"stop-254kg","State":""}
{"level":"info","ts":"2026-11-03T15:08:33.132Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"b85330bd-cae8-44c1-ab6c-edb0e41155cb","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:08:27Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:08:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:08:33Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:08:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:08:33.132Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"b85330bd-cae8-44c1-ab6c-edb0e41155cb","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:08:33.193Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"8e3df60b-bca1-4b1d-96a9-76cdaae81fd5","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:08:33.193Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"8e3df60b-bca1-4b1d-96a9-76cdaae81fd5","name":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:08:33.193Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"8e3df60b-bca1-4b1d-96a9-76cdaae81fd5","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:08:33.193Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"8e3df60b-bca1-4b1d-96a9-76cdaae81fd5"}
{"level":"info","ts":"2026-11-03T15:08:33.193Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"8e3df60b-bca1-4b1d-96a9-76cdaae81fd5","RayCluster":"stop-254kg","State":""}
{"level":"info","ts":"2026-11-03T15:08:33.193Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"8e3df60b-bca1-4b1d-96a9-76cdaae81fd5","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:08:33Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:08:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:08:33Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:08:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:08:35.132Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"f369aac1-5dfe-4f97-a86b-d542a6eeee4e"}
{"level":"info","ts":"2026-11-03T15:08:35.132Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"f369aac1-5dfe-4f97-a86b-d542a6eeee4e"}
{"level":"info","ts":"2026-11-03T15:08:35.132Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"f369aac1-5dfe-4f97-a86b-d542a6eeee4e","1 head service found":"stop-254kg-head-svc"}
{"level":"info","ts":"2026-11-03T15:08:35.132Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"f369aac1-5dfe-4f97-a86b-d542a6eeee4e","Found 1 head Pod":"stop-254kg-head-4nd85","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:08:35.132Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"f369aac1-5dfe-4f97-a86b-d542a6eeee4e","head Pod":"stop-254kg-head-4nd85","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-254kg-head-4nd85. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:08:35.132Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"f369aac1-5dfe-4f97-a86b-d542a6eeee4e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:08:35.132Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"f369aac1-5dfe-4f97-a86b-d542a6eeee4e","worker Pod":"stop-254kg-small-group-worker-nhg8h","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-254kg-small-group-worker-nhg8h. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:08:35.132Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"f369aac1-5dfe-4f97-a86b-d542a6eeee4e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:08:35.132Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"f369aac1-5dfe-4f97-a86b-d542a6eeee4e","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:08:35.132Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"f369aac1-5dfe-4f97-a86b-d542a6eeee4e","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:08:35.132Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"f369aac1-5dfe-4f97-a86b-d542a6eeee4e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:08:35.132Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"f369aac1-5dfe-4f97-a86b-d542a6eeee4e","podName":"stop-254kg-small-group-worker-nhg8h","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:07:04Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:08:35.133Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"f369aac1-5dfe-4f97-a86b-d542a6eeee4e","podName":"stop-254kg-small-group-worker-nhg8h","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:07:04Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:08:35.133Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"f369aac1-5dfe-4f97-a86b-d542a6eeee4e","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:08:35.133Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"f369aac1-5dfe-4f97-a86b-d542a6eeee4e","seconds":300}
{"level":"info","ts":"2026-11-03T15:08:36.194Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"8964286f-db4a-402f-afea-f102485d0c4b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:08:36.194Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"8964286f-db4a-402f-afea-f102485d0c4b","name":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:08:36.194Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"8964286f-db4a-402f-afea-f102485d0c4b","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:08:36.194Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"8964286f-db4a-402f-afea-f102485d0c4b"}
{"level":"info","ts":"2026-11-03T15:08:36.194Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"8964286f-db4a-402f-afea-f102485d0c4b","RayCluster":"stop-254kg","State":""}
{"level":"info","ts":"2026-11-03T15:08:36.194Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"8964286f-db4a-402f-afea-f102485d0c4b","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:08:33Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:08:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:08:33Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:08:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:08:39.195Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"9bb372a4-e03f-40dd-9bd1-da9c5564c815","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:08:39.195Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"9bb372a4-e03f-40dd-9bd1-da9c5564c815","name":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:08:39.195Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"9bb372a4-e03f-40dd-9bd1-da9c5564c815","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:08:39.195Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"9bb372a4-e03f-40dd-9bd1-da9c5564c815"}
{"level":"info","ts":"2026-11-03T15:08:39.195Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"9bb372a4-e03f-40dd-9bd1-da9c5564c815","RayCluster":"stop-254kg","State":""}
{"level":"info","ts":"2026-11-03T15:08:39.195Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"9bb372a4-e03f-40dd-9bd1-da9c5564c815","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:08:33Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:08:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:08:33Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:08:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:08:42.196Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"a6cd8ed2-9f19-4210-9136-c8c75b8e8488","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:08:42.196Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"a6cd8ed2-9f19-4210-9136-c8c75b8e8488","name":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:08:42.196Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"a6cd8ed2-9f19-4210-9136-c8c75b8e8488","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:08:42.196Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"a6cd8ed2-9f19-4210-9136-c8c75b8e8488"}
{"level":"info","ts":"2026-11-03T15:08:42.196Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"a6cd8ed2-9f19-4210-9136-c8c75b8e8488","RayCluster":"stop-254kg","State":""}
{"level":"info","ts":"2026-11-03T15:08:42.196Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"a6cd8ed2-9f19-4210-9136-c8c75b8e8488","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:08:33Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:08:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:08:33Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:08:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:08:44.101Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"5c3ea045-2b79-4bfa-add9-9c48b6d2574b"}
{"level":"info","ts":"2026-11-03T15:08:44.101Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"5c3ea045-2b79-4bfa-add9-9c48b6d2574b"}
{"level":"info","ts":"2026-11-03T15:08:44.101Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"5c3ea045-2b79-4bfa-add9-9c48b6d2574b","1 head service found":"stop-254kg-head-svc"}
{"level":"info","ts":"2026-11-03T15:08:44.101Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"5c3ea045-2b79-4bfa-add9-9c48b6d2574b","Found 1 head Pod":"stop-254kg-head-4nd85","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:08:44.101Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"5c3ea045-2b79-4bfa-add9-9c48b6d2574b","head Pod":"stop-254kg-head-4nd85","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-254kg-head-4nd85. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:08:44.101Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"5c3ea045-2b79-4bfa-add9-9c48b6d2574b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:08:44.101Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"5c3ea045-2b79-4bfa-add9-9c48b6d2574b","worker Pod":"stop-254kg-small-group-worker-nhg8h","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-254kg-small-group-worker-nhg8h. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:08:44.101Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"5c3ea045-2b79-4bfa-add9-9c48b6d2574b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:08:44.101Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"5c3ea045-2b79-4bfa-add9-9c48b6d2574b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:08:44.101Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"5c3ea045-2b79-4bfa-add9-9c48b6d2574b","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:08:44.101Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"5c3ea045-2b79-4bfa-add9-9c48b6d2574b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:08:44.102Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"5c3ea045-2b79-4bfa-add9-9c48b6d2574b","name":"stop-254kg","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:08:33Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:08:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:08:44Z","stateTransitionTimes":{"ready":"2026-11-03T15:08:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:08:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:08:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:08:44.164Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"e89ef7da-524b-4b27-9e66-6fcbf2fd47bc","cluster":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:08:44.164Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"0a6c7053-009b-4289-8216-405e9b5e7224","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:08:44.164Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"0a6c7053-009b-4289-8216-405e9b5e7224","name":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:08:44.164Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"0a6c7053-009b-4289-8216-405e9b5e7224","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:08:44.164Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"0a6c7053-009b-4289-8216-405e9b5e7224"}
{"level":"info","ts":"2026-11-03T15:08:44.164Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"0a6c7053-009b-4289-8216-405e9b5e7224","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:08:33Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:08:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","dashboardURL":"stop-254kg-head-svc.test-ns-mvz6h.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:08:33Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:08:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:08:44.164Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"0a6c7053-009b-4289-8216-405e9b5e7224","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-11-03T15:08:44.224Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"287d951d-3bda-4101-87d2-eaaf3e8f2763","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:08:44.224Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"287d951d-3bda-4101-87d2-eaaf3e8f2763","name":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:08:44.224Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"287d951d-3bda-4101-87d2-eaaf3e8f2763","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:08:44.224Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"287d951d-3bda-4101-87d2-eaaf3e8f2763"}
{"level":"info","ts":"2026-11-03T15:08:44.224Z","logger":"controllers.RayJob","msg":"Create a Kubernetes Service","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"287d951d-3bda-4101-87d2-eaaf3e8f2763","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:08:44.294Z","logger":"controllers.RayJob","msg":"The Ray job was not found. Submit a Ray job via an HTTP request.","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"287d951d-3bda-4101-87d2-eaaf3e8f2763","JobId":"stop-6knsg"}
{"level":"info","ts":"2026-11-03T15:08:45.139Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"64f20fe1-ba98-4a39-a446-6f9360ef188e","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:08:45.139Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"64f20fe1-ba98-4a39-a446-6f9360ef188e","name":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:08:45.139Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"64f20fe1-ba98-4a39-a446-6f9360ef188e","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:08:45.139Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"64f20fe1-ba98-4a39-a446-6f9360ef188e"}
{"level":"info","ts":"2026-11-03T15:08:45.139Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"64f20fe1-ba98-4a39-a446-6f9360ef188e","rayCluster":"stop-254kg","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:08:45.142Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"64f20fe1-ba98-4a39-a446-6f9360ef188e","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","dashboardURL":"stop-254kg-head-svc.test-ns-mvz6h.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:08:33Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:08:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:08:44Z"},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","dashboardURL":"stop-254kg-head-svc.test-ns-mvz6h.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:08:44Z","stateTransitionTimes":{"ready":"2026-11-03T15:08:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:08:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:08:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:08:45.142Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"64f20fe1-ba98-4a39-a446-6f9360ef188e","old JobStatus":"","new JobStatus":"PENDING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-11-03T15:08:45.203Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"2754cfc3-caa9-470d-ab17-da0103a8f6fb","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:08:45.203Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"2754cfc3-caa9-470d-ab17-da0103a8f6fb","name":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:08:45.203Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"2754cfc3-caa9-470d-ab17-da0103a8f6fb","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:08:45.203Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"2754cfc3-caa9-470d-ab17-da0103a8f6fb"}
{"level":"info","ts":"2026-11-03T15:08:45.203Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"2754cfc3-caa9-470d-ab17-da0103a8f6fb","rayCluster":"stop-254kg","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:08:45.206Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"2754cfc3-caa9-470d-ab17-da0103a8f6fb","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:08:44Z"},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","dashboardURL":"stop-254kg-head-svc.test-ns-mvz6h.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:08:44Z","stateTransitionTimes":{"ready":"2026-11-03T15:08:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:08:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:08:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:08:44Z"},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","dashboardURL":"stop-254kg-head-svc.test-ns-mvz6h.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:08:44Z","stateTransitionTimes":{"ready":"2026-11-03T15:08:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:08:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:08:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:08:46.164Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"d527c09b-3045-49d9-a22e-5eda35ed776f"}
{"level":"info","ts":"2026-11-03T15:08:46.164Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"d527c09b-3045-49d9-a22e-5eda35ed776f"}
{"level":"info","ts":"2026-11-03T15:08:46.164Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"d527c09b-3045-49d9-a22e-5eda35ed776f","1 head service found":"stop-254kg-head-svc"}
{"level":"info","ts":"2026-11-03T15:08:46.165Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"d527c09b-3045-49d9-a22e-5eda35ed776f","Found 1 head Pod":"stop-254kg-head-4nd85","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:08:46.165Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"d527c09b-3045-49d9-a22e-5eda35ed776f","head Pod":"stop-254kg-head-4nd85","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-254kg-head-4nd85. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:08:46.165Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"d527c09b-3045-49d9-a22e-5eda35ed776f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:08:46.165Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"d527c09b-3045-49d9-a22e-5eda35ed776f","worker Pod":"stop-254kg-small-group-worker-nhg8h","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-254kg-small-group-worker-nhg8h. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:08:46.165Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"d527c09b-3045-49d9-a22e-5eda35ed776f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:08:46.165Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"d527c09b-3045-49d9-a22e-5eda35ed776f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:08:46.165Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"d527c09b-3045-49d9-a22e-5eda35ed776f","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:08:46.165Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"d527c09b-3045-49d9-a22e-5eda35ed776f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:08:46.165Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"d527c09b-3045-49d9-a22e-5eda35ed776f","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:08:46.165Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"d527c09b-3045-49d9-a22e-5eda35ed776f","seconds":300}
{"level":"info","ts":"2026-11-03T15:08:48.207Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"7396dd76-fa79-428f-b384-a94b34aac8cc","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:08:48.207Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"7396dd76-fa79-428f-b384-a94b34aac8cc","name":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:08:48.207Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"7396dd76-fa79-428f-b384-a94b34aac8cc","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:08:48.207Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"7396dd76-fa79-428f-b384-a94b34aac8cc"}
{"level":"info","ts":"2026-11-03T15:08:48.207Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"7396dd76-fa79-428f-b384-a94b34aac8cc","rayCluster":"stop-254kg","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:08:48.210Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"7396dd76-fa79-428f-b384-a94b34aac8cc","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:08:44Z"},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","dashboardURL":"stop-254kg-head-svc.test-ns-mvz6h.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:08:44Z","stateTransitionTimes":{"ready":"2026-11-03T15:08:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:08:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:08:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:08:44Z"},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","dashboardURL":"stop-254kg-head-svc.test-ns-mvz6h.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:08:44Z","stateTransitionTimes":{"ready":"2026-11-03T15:08:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:08:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:08:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:08:48.210Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"7396dd76-fa79-428f-b384-a94b34aac8cc","old JobStatus":"PENDING","new JobStatus":"RUNNING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-11-03T15:08:48.307Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"49e4af22-c23b-42b7-b20c-30e0774b3d2d","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:08:48.307Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"49e4af22-c23b-42b7-b20c-30e0774b3d2d","name":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:08:48.307Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"49e4af22-c23b-42b7-b20c-30e0774b3d2d","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:08:48.307Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"49e4af22-c23b-42b7-b20c-30e0774b3d2d"}
{"level":"info","ts":"2026-11-03T15:08:48.307Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"49e4af22-c23b-42b7-b20c-30e0774b3d2d","rayCluster":"stop-254kg","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:08:48.310Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"49e4af22-c23b-42b7-b20c-30e0774b3d2d","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:08:44Z"},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","dashboardURL":"stop-254kg-head-svc.test-ns-mvz6h.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:08:44Z","stateTransitionTimes":{"ready":"2026-11-03T15:08:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:08:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:08:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:08:44Z"},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","dashboardURL":"stop-254kg-head-svc.test-ns-mvz6h.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:08:44Z","stateTransitionTimes":{"ready":"2026-11-03T15:08:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:08:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:08:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:08:51.311Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"8baf608f-556f-49b1-b01e-e6141f71c83d","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:08:51.311Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"8baf608f-556f-49b1-b01e-e6141f71c83d","name":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:08:51.311Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"8baf608f-556f-49b1-b01e-e6141f71c83d","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:08:51.311Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"8baf608f-556f-49b1-b01e-e6141f71c83d"}
{"level":"info","ts":"2026-11-03T15:08:51.311Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"8baf608f-556f-49b1-b01e-e6141f71c83d","rayCluster":"stop-254kg","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:08:51.314Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"8baf608f-556f-49b1-b01e-e6141f71c83d","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:08:44Z"},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","dashboardURL":"stop-254kg-head-svc.test-ns-mvz6h.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:08:44Z","stateTransitionTimes":{"ready":"2026-11-03T15:08:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:08:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:08:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:08:44Z"},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","dashboardURL":"stop-254kg-head-svc.test-ns-mvz6h.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:08:44Z","stateTransitionTimes":{"ready":"2026-11-03T15:08:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:08:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:08:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:08:54.315Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"2a6ec69f-19f8-4206-8b69-cb21ad378ce2","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:08:54.315Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"2a6ec69f-19f8-4206-8b69-cb21ad378ce2","name":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:08:54.315Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"2a6ec69f-19f8-4206-8b69-cb21ad378ce2","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:08:54.315Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"2a6ec69f-19f8-4206-8b69-cb21ad378ce2"}
{"level":"info","ts":"2026-11-03T15:08:54.315Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"2a6ec69f-19f8-4206-8b69-cb21ad378ce2","rayCluster":"stop-254kg","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:08:54.318Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"2a6ec69f-19f8-4206-8b69-cb21ad378ce2","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:08:44Z"},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","dashboardURL":"stop-254kg-head-svc.test-ns-mvz6h.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:08:44Z","stateTransitionTimes":{"ready":"2026-11-03T15:08:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:08:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:08:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:08:44Z"},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","dashboardURL":"stop-254kg-head-svc.test-ns-mvz6h.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:08:44Z","stateTransitionTimes":{"ready":"2026-11-03T15:08:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:08:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:08:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:08:57.319Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"a48e7882-5886-46c3-98bb-7fc6d898edff","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:08:57.320Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"a48e7882-5886-46c3-98bb-7fc6d898edff","name":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:08:57.320Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"a48e7882-5886-46c3-98bb-7fc6d898edff","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:08:57.320Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"a48e7882-5886-46c3-98bb-7fc6d898edff"}
{"level":"info","ts":"2026-11-03T15:08:57.320Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"a48e7882-5886-46c3-98bb-7fc6d898edff","rayCluster":"stop-254kg","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:08:57.323Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"a48e7882-5886-46c3-98bb-7fc6d898edff","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:08:44Z"},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","dashboardURL":"stop-254kg-head-svc.test-ns-mvz6h.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:08:44Z","stateTransitionTimes":{"ready":"2026-11-03T15:08:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:08:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:08:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:08:44Z"},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","dashboardURL":"stop-254kg-head-svc.test-ns-mvz6h.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:08:44Z","stateTransitionTimes":{"ready":"2026-11-03T15:08:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:08:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:08:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:09:00.324Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"cb2e07e9-2e81-4652-84ac-ce1569321ad8","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:09:00.324Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"cb2e07e9-2e81-4652-84ac-ce1569321ad8","name":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:09:00.324Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"cb2e07e9-2e81-4652-84ac-ce1569321ad8","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:09:00.324Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"cb2e07e9-2e81-4652-84ac-ce1569321ad8"}
{"level":"info","ts":"2026-11-03T15:09:00.324Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"cb2e07e9-2e81-4652-84ac-ce1569321ad8","rayCluster":"stop-254kg","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:09:00.327Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"cb2e07e9-2e81-4652-84ac-ce1569321ad8","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:08:44Z"},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","dashboardURL":"stop-254kg-head-svc.test-ns-mvz6h.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:08:44Z","stateTransitionTimes":{"ready":"2026-11-03T15:08:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:08:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:08:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:08:44Z"},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","dashboardURL":"stop-254kg-head-svc.test-ns-mvz6h.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:08:44Z","stateTransitionTimes":{"ready":"2026-11-03T15:08:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:08:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:08:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:09:03.328Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"34f62033-932a-48b5-b91d-9876280be3cc","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:09:03.328Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"34f62033-932a-48b5-b91d-9876280be3cc","name":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:09:03.328Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"34f62033-932a-48b5-b91d-9876280be3cc","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:09:03.328Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"34f62033-932a-48b5-b91d-9876280be3cc"}
{"level":"info","ts":"2026-11-03T15:09:03.328Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"34f62033-932a-48b5-b91d-9876280be3cc","rayCluster":"stop-254kg","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:09:03.331Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"34f62033-932a-48b5-b91d-9876280be3cc","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:08:44Z"},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","dashboardURL":"stop-254kg-head-svc.test-ns-mvz6h.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:08:44Z","stateTransitionTimes":{"ready":"2026-11-03T15:08:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:08:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:08:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:08:44Z"},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","dashboardURL":"stop-254kg-head-svc.test-ns-mvz6h.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:08:44Z","stateTransitionTimes":{"ready":"2026-11-03T15:08:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:08:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:08:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:09:06.331Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"0a262edb-d623-4db4-9282-0cdbc384ae2f","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:09:06.332Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"0a262edb-d623-4db4-9282-0cdbc384ae2f","name":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:09:06.332Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"0a262edb-d623-4db4-9282-0cdbc384ae2f","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:09:06.332Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"0a262edb-d623-4db4-9282-0cdbc384ae2f"}
{"level":"info","ts":"2026-11-03T15:09:06.332Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"0a262edb-d623-4db4-9282-0cdbc384ae2f","rayCluster":"stop-254kg","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:09:06.335Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"0a262edb-d623-4db4-9282-0cdbc384ae2f","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:08:44Z"},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","dashboardURL":"stop-254kg-head-svc.test-ns-mvz6h.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:08:44Z","stateTransitionTimes":{"ready":"2026-11-03T15:08:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:08:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:08:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:08:44Z"},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","dashboardURL":"stop-254kg-head-svc.test-ns-mvz6h.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:08:44Z","stateTransitionTimes":{"ready":"2026-11-03T15:08:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:08:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:08:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:09:09.336Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"cb5d532d-1e15-45f1-83b7-6bbcd8428ab0","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:09:09.336Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"cb5d532d-1e15-45f1-83b7-6bbcd8428ab0","name":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:09:09.336Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"cb5d532d-1e15-45f1-83b7-6bbcd8428ab0","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:09:09.336Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"cb5d532d-1e15-45f1-83b7-6bbcd8428ab0"}
{"level":"info","ts":"2026-11-03T15:09:09.336Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"cb5d532d-1e15-45f1-83b7-6bbcd8428ab0","rayCluster":"stop-254kg","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:09:09.339Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"cb5d532d-1e15-45f1-83b7-6bbcd8428ab0","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:08:44Z"},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","dashboardURL":"stop-254kg-head-svc.test-ns-mvz6h.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:08:44Z","stateTransitionTimes":{"ready":"2026-11-03T15:08:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:08:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:08:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:08:44Z","endTime":"2026-11-03T15:09:07Z"},"jobId":"stop-6knsg","rayClusterName":"stop-254kg","dashboardURL":"stop-254kg-head-svc.test-ns-mvz6h.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Complete","message":"Job was intentionally stopped.","startTime":"2026-11-03T15:06:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:08:44Z","stateTransitionTimes":{"ready":"2026-11-03T15:08:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.33","serviceIP":"10.132.0.33","podName":"stop-254kg-head-4nd85","serviceName":"stop-254kg-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:08:27Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:08:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:06:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:06:58Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:09:09.339Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"cb5d532d-1e15-45f1-83b7-6bbcd8428ab0","old JobStatus":"RUNNING","new JobStatus":"STOPPED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Complete"}
{"level":"info","ts":"2026-11-03T15:09:09.436Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"03393b38-d282-41f0-b403-88421a3484d5","JobStatus":"STOPPED","JobDeploymentStatus":"Complete","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:09:09.488Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"8da554b4-9110-4d7a-b3a6-3cc0b2bd05a6","DeletionTimestamp":"2026-11-03 15:09:09 +0000 UTC"}
{"level":"info","ts":"2026-11-03T15:09:09.488Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"8da554b4-9110-4d7a-b3a6-3cc0b2bd05a6","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-11-03T15:09:09.549Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"c2e767ec-833d-48ca-82a0-7389165bbcc4","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:09:09.549Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"c2e767ec-833d-48ca-82a0-7389165bbcc4","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-11-03T15:09:09.571Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"stop-254kg","reconcileID":"93ad8d60-aa9a-40a2-8413-08fe1d8d50f1","namespacedName":{"name":"stop-254kg","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:09:09.571Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"stop-254kg","reconcileID":"93ad8d60-aa9a-40a2-8413-08fe1d8d50f1","cluster":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:09:09.571Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"stop-254kg","reconcileID":"93ad8d60-aa9a-40a2-8413-08fe1d8d50f1","cluster":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:09:09.571Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"stop-254kg","reconcileID":"93ad8d60-aa9a-40a2-8413-08fe1d8d50f1","namespace":"test-ns-mvz6h","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-11-03T15:09:09.571Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"reconcileID":"d943cb3c-f547-486b-b1ac-8981e5838b29"}
{"level":"info","ts":"2026-11-03T15:09:09.607Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"c2e767ec-833d-48ca-82a0-7389165bbcc4"}
{"level":"info","ts":"2026-11-03T15:09:09.607Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"c2e767ec-833d-48ca-82a0-7389165bbcc4","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-11-03T15:09:09.607Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"c2e767ec-833d-48ca-82a0-7389165bbcc4","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:09:09.607Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"c2e767ec-833d-48ca-82a0-7389165bbcc4","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:09:09.623Z","logger":"authentication-controller","msg":"Deleted shared ReferenceGrant (last cluster with auth)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"stop-254kg","reconcileID":"93ad8d60-aa9a-40a2-8413-08fe1d8d50f1","name":"kuberay-gateway-access","namespace":"test-ns-mvz6h"}
{"level":"info","ts":"2026-11-03T15:09:09.665Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"9d47bf1b-5e0c-4b06-ad09-ad8cc2fc0274"}
{"level":"info","ts":"2026-11-03T15:09:09.665Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"d5a1e633-e001-49c0-a3c4-1a41bec6d511","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:09:09.666Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"d5a1e633-e001-49c0-a3c4-1a41bec6d511","name":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:09:09.666Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"d5a1e633-e001-49c0-a3c4-1a41bec6d511","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:09:09.666Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"d5a1e633-e001-49c0-a3c4-1a41bec6d511","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:09:09.675Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"stop-254kg","reconcileID":"93ad8d60-aa9a-40a2-8413-08fe1d8d50f1","name":"test-ns-mvz6h-stop-254kg","namespace":"opendatahub"}
{"level":"info","ts":"2026-11-03T15:09:09.727Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"stop-254kg","reconcileID":"93ad8d60-aa9a-40a2-8413-08fe1d8d50f1","configMap":"kube-rbac-proxy-config-stop-254kg"}
{"level":"info","ts":"2026-11-03T15:09:09.778Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"stop-254kg","reconcileID":"93ad8d60-aa9a-40a2-8413-08fe1d8d50f1","serviceAccount":"stop-254kg-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-11-03T15:09:09.914Z","logger":"raycluster-resource","msg":"default","name":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:09:09.914Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"}
{"level":"info","ts":"2026-11-03T15:09:09.944Z","logger":"raycluster-resource","msg":"default","name":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:09:09.944Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"stop-254kg","namespace":"test-ns-mvz6h"}
{"level":"info","ts":"2026-11-03T15:09:10.101Z","logger":"raycluster-resource","msg":"validate create","name":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:09:10.124Z","logger":"raycluster-resource","msg":"validate update","name":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:09:10.155Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"d5a1e633-e001-49c0-a3c4-1a41bec6d511","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:09:10.155Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"d5a1e633-e001-49c0-a3c4-1a41bec6d511"}
{"level":"info","ts":"2026-11-03T15:09:10.155Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"d5a1e633-e001-49c0-a3c4-1a41bec6d511","RayCluster":"counter-auth-kk8bt","State":""}
{"level":"info","ts":"2026-11-03T15:09:10.155Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"d5a1e633-e001-49c0-a3c4-1a41bec6d511","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:09:10.155Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"86b62ffe-da41-474b-b808-a0419f30a6d2"}
{"level":"info","ts":"2026-11-03T15:09:10.155Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"86b62ffe-da41-474b-b808-a0419f30a6d2"}
{"level":"info","ts":"2026-11-03T15:09:10.155Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"7b55ea95-3cb3-4569-82a1-b9943872755b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:09:10.155Z","logger":"controllers.NetworkPolicy","msg":"Reconciling NetworkPolicies for RayCluster","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"9dfffe03-bb5a-4795-ac58-26112ca195dd","cluster":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"}
{"level":"info","ts":"2026-11-03T15:09:10.155Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"c414c77e-036f-4cd5-b77c-69ac2e4dc2a8","cluster":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:09:10.155Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"7b55ea95-3cb3-4569-82a1-b9943872755b","name":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:09:10.155Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"7b55ea95-3cb3-4569-82a1-b9943872755b","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:09:10.156Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"7b55ea95-3cb3-4569-82a1-b9943872755b"}
{"level":"info","ts":"2026-11-03T15:09:10.156Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"7b55ea95-3cb3-4569-82a1-b9943872755b","RayCluster":"counter-auth-kk8bt","State":""}
{"level":"info","ts":"2026-11-03T15:09:10.156Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"7b55ea95-3cb3-4569-82a1-b9943872755b","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:09:10.179Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"stop-254kg","reconcileID":"93ad8d60-aa9a-40a2-8413-08fe1d8d50f1","cluster":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:09:10.179Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"stop-254kg","reconcileID":"75fee038-ea3b-486f-b811-86c2813426f2","namespacedName":{"name":"stop-254kg","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:09:10.179Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"stop-254kg","reconcileID":"75fee038-ea3b-486f-b811-86c2813426f2","cluster":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:09:10.179Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"stop-254kg","reconcileID":"75fee038-ea3b-486f-b811-86c2813426f2","cluster":"stop-254kg"}
{"level":"info","ts":"2026-11-03T15:09:10.179Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"stop-254kg","reconcileID":"75fee038-ea3b-486f-b811-86c2813426f2","namespace":"test-ns-mvz6h","otherAuthClusters":1,"action":"will retain"}
{"level":"info","ts":"2026-11-03T15:09:10.179Z","logger":"authentication-controller","msg":"ReferenceGrant retained (other clusters with auth exist)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"stop-254kg","reconcileID":"75fee038-ea3b-486f-b811-86c2813426f2","name":"kuberay-gateway-access","namespace":"test-ns-mvz6h","remainingClusters":1}
{"level":"info","ts":"2026-11-03T15:09:10.179Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"stop","namespace":"test-ns-mvz6h"},"reconcileID":"a4651a33-de46-4316-95a7-3fd190be46bc"}
{"level":"info","ts":"2026-11-03T15:09:10.226Z","logger":"controllers.NetworkPolicy","msg":"Successfully created NetworkPolicy","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"9dfffe03-bb5a-4795-ac58-26112ca195dd","name":"counter-auth-kk8bt-head"}
{"level":"error","ts":"2026-11-03T15:09:10.229Z","logger":"authentication-controller","msg":"Failed to remove finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"stop-254kg","reconcileID":"75fee038-ea3b-486f-b811-86c2813426f2","error":"rayclusters.ray.io \"stop-254kg\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleDeletion\n\t/workspace/controllers/ray/authentication_controller.go:195\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:09:10.230Z","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"stop-254kg","reconcileID":"75fee038-ea3b-486f-b811-86c2813426f2"}
{"level":"error","ts":"2026-11-03T15:09:10.230Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"stop-254kg","reconcileID":"75fee038-ea3b-486f-b811-86c2813426f2","error":"rayclusters.ray.io \"stop-254kg\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:09:10.230Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-auth-kk8bt","reconcileID":"0a004272-86d9-462f-97d5-81b1574800ca","namespacedName":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:09:10.230Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-auth-kk8bt","reconcileID":"0a004272-86d9-462f-97d5-81b1574800ca","mode":"IntegratedOAuth","cluster":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:09:10.230Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-auth-kk8bt","reconcileID":"0a004272-86d9-462f-97d5-81b1574800ca","cluster":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:09:10.230Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-auth-kk8bt","reconcileID":"0a004272-86d9-462f-97d5-81b1574800ca","cluster":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:09:10.259Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"86b62ffe-da41-474b-b808-a0419f30a6d2","name":"counter-auth-kk8bt-head-svc"}
{"level":"info","ts":"2026-11-03T15:09:10.259Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"86b62ffe-da41-474b-b808-a0419f30a6d2"}
{"level":"info","ts":"2026-11-03T15:09:10.278Z","logger":"controllers.NetworkPolicy","msg":"Successfully created NetworkPolicy","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"9dfffe03-bb5a-4795-ac58-26112ca195dd","name":"counter-auth-kk8bt-workers-small-group"}
{"level":"info","ts":"2026-11-03T15:09:10.278Z","logger":"controllers.NetworkPolicy","msg":"Successfully reconciled NetworkPolicies for RayCluster","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"9dfffe03-bb5a-4795-ac58-26112ca195dd","cluster":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"}
{"level":"info","ts":"2026-11-03T15:09:10.279Z","logger":"controllers.NetworkPolicy","msg":"Reconciling NetworkPolicies for RayCluster","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"19c71b95-247c-4d5f-af5a-20b9a6477d5f","cluster":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"}
{"level":"info","ts":"2026-11-03T15:09:10.282Z","logger":"raycluster-resource","msg":"default","name":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:09:10.282Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"}
{"level":"info","ts":"2026-11-03T15:09:10.308Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"86b62ffe-da41-474b-b808-a0419f30a6d2","error":"certificate ray-head-cert-counter-auth-kk8bt not found: certificates.cert-manager.io \"ray-head-cert-counter-auth-kk8bt\" not found"}
{"level":"error","ts":"2026-11-03T15:09:10.308Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"86b62ffe-da41-474b-b808-a0419f30a6d2","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-counter-auth-kk8bt not found: certificates.cert-manager.io \"ray-head-cert-counter-auth-kk8bt\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:09:10.308Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"86b62ffe-da41-474b-b808-a0419f30a6d2","error":"mTLS secrets not ready: certificate ray-head-cert-counter-auth-kk8bt not found: certificates.cert-manager.io \"ray-head-cert-counter-auth-kk8bt\" not found"}
{"level":"info","ts":"2026-11-03T15:09:10.308Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"86b62ffe-da41-474b-b808-a0419f30a6d2","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"counter-auth-kk8bt","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:09:10.308Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"86b62ffe-da41-474b-b808-a0419f30a6d2","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"counter-auth-kk8bt","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:09:10.308Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"86b62ffe-da41-474b-b808-a0419f30a6d2","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"counter-auth-kk8bt","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:09:10.308Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"86b62ffe-da41-474b-b808-a0419f30a6d2","name":"counter-auth-kk8bt","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:09:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:09:10.339Z","logger":"raycluster-resource","msg":"validate update","name":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:09:10.370Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"86b62ffe-da41-474b-b808-a0419f30a6d2"}
{"level":"error","ts":"2026-11-03T15:09:10.370Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"86b62ffe-da41-474b-b808-a0419f30a6d2","error":"mTLS secrets not ready: certificate ray-head-cert-counter-auth-kk8bt not found: certificates.cert-manager.io \"ray-head-cert-counter-auth-kk8bt\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:09:10.370Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"e3162694-33e0-4268-bd91-c377e3b96891"}
{"level":"info","ts":"2026-11-03T15:09:10.370Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"e3162694-33e0-4268-bd91-c377e3b96891"}
{"level":"info","ts":"2026-11-03T15:09:10.370Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"e3162694-33e0-4268-bd91-c377e3b96891","1 head service found":"counter-auth-kk8bt-head-svc"}
{"level":"info","ts":"2026-11-03T15:09:10.370Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"e3162694-33e0-4268-bd91-c377e3b96891"}
{"level":"info","ts":"2026-11-03T15:09:10.370Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"dbe55365-c5bb-4c2e-9825-5aa98ba2ccac","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:09:10.370Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"dbe55365-c5bb-4c2e-9825-5aa98ba2ccac","name":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:09:10.370Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"dbe55365-c5bb-4c2e-9825-5aa98ba2ccac","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:09:10.370Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"dbe55365-c5bb-4c2e-9825-5aa98ba2ccac"}
{"level":"info","ts":"2026-11-03T15:09:10.370Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"dbe55365-c5bb-4c2e-9825-5aa98ba2ccac","RayCluster":"counter-auth-kk8bt","State":""}
{"level":"info","ts":"2026-11-03T15:09:10.370Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"dbe55365-c5bb-4c2e-9825-5aa98ba2ccac","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:09:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:09:10.370Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"dbe55365-c5bb-4c2e-9825-5aa98ba2ccac","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:09:10.383Z","logger":"controllers.NetworkPolicy","msg":"Successfully reconciled NetworkPolicies for RayCluster","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"19c71b95-247c-4d5f-af5a-20b9a6477d5f","cluster":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"}
{"level":"info","ts":"2026-11-03T15:09:10.395Z","logger":"raycluster-resource","msg":"default","name":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:09:10.395Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"}
{"level":"info","ts":"2026-11-03T15:09:10.419Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"e3162694-33e0-4268-bd91-c377e3b96891","error":"certificate ray-head-cert-counter-auth-kk8bt not found: certificates.cert-manager.io \"ray-head-cert-counter-auth-kk8bt\" not found"}
{"level":"error","ts":"2026-11-03T15:09:10.419Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"e3162694-33e0-4268-bd91-c377e3b96891","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-counter-auth-kk8bt not found: certificates.cert-manager.io \"ray-head-cert-counter-auth-kk8bt\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:09:10.419Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"e3162694-33e0-4268-bd91-c377e3b96891","error":"mTLS secrets not ready: certificate ray-head-cert-counter-auth-kk8bt not found: certificates.cert-manager.io \"ray-head-cert-counter-auth-kk8bt\" not found"}
{"level":"info","ts":"2026-11-03T15:09:10.419Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"e3162694-33e0-4268-bd91-c377e3b96891","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"counter-auth-kk8bt","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:09:10.420Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"e3162694-33e0-4268-bd91-c377e3b96891","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"counter-auth-kk8bt","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:09:10.420Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"e3162694-33e0-4268-bd91-c377e3b96891","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"counter-auth-kk8bt","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:09:10.420Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"e3162694-33e0-4268-bd91-c377e3b96891","name":"counter-auth-kk8bt","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:09:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:09:10.429Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"1d241e32-c613-44dd-96c3-97be5def33f6","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:09:10.429Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"1d241e32-c613-44dd-96c3-97be5def33f6","name":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:09:10.429Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"1d241e32-c613-44dd-96c3-97be5def33f6","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:09:10.429Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"1d241e32-c613-44dd-96c3-97be5def33f6"}
{"level":"info","ts":"2026-11-03T15:09:10.429Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"1d241e32-c613-44dd-96c3-97be5def33f6","RayCluster":"counter-auth-kk8bt","State":""}
{"level":"info","ts":"2026-11-03T15:09:10.429Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"1d241e32-c613-44dd-96c3-97be5def33f6","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:09:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:09:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"error","ts":"2026-11-03T15:09:10.444Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-auth-kk8bt","reconcileID":"0a004272-86d9-462f-97d5-81b1574800ca","error":"failed to add finalizer: Operation cannot be fulfilled on rayclusters.ray.io \"counter-auth-kk8bt\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:167\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:09:10.444Z","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-auth-kk8bt","reconcileID":"0a004272-86d9-462f-97d5-81b1574800ca"}
{"level":"error","ts":"2026-11-03T15:09:10.444Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-auth-kk8bt","reconcileID":"0a004272-86d9-462f-97d5-81b1574800ca","error":"failed to add finalizer: Operation cannot be fulfilled on rayclusters.ray.io \"counter-auth-kk8bt\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:09:10.444Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"stop-254kg","reconcileID":"7debee3d-dd5c-4f5b-87a5-e197c6f6163d","namespacedName":{"name":"stop-254kg","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:09:10.444Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"stop-254kg","reconcileID":"7debee3d-dd5c-4f5b-87a5-e197c6f6163d"}
{"level":"info","ts":"2026-11-03T15:09:10.444Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"stop-254kg","reconcileID":"7debee3d-dd5c-4f5b-87a5-e197c6f6163d","namespace":"test-ns-mvz6h","authClusters":1}
{"level":"info","ts":"2026-11-03T15:09:10.444Z","logger":"authentication-controller","msg":"ReferenceGrant not orphaned (clusters with auth exist)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-254kg","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"stop-254kg","reconcileID":"7debee3d-dd5c-4f5b-87a5-e197c6f6163d","namespace":"test-ns-mvz6h","authClusters":1}
{"level":"info","ts":"2026-11-03T15:09:10.444Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-auth-kk8bt","reconcileID":"9d1700f5-8f1d-4aaa-8c5a-a9ea01b51ee7","namespacedName":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:09:10.444Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-auth-kk8bt","reconcileID":"9d1700f5-8f1d-4aaa-8c5a-a9ea01b51ee7","mode":"IntegratedOAuth","cluster":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:09:10.444Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-auth-kk8bt","reconcileID":"9d1700f5-8f1d-4aaa-8c5a-a9ea01b51ee7","cluster":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:09:10.444Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-auth-kk8bt","reconcileID":"9d1700f5-8f1d-4aaa-8c5a-a9ea01b51ee7","cluster":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:09:10.474Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"e3162694-33e0-4268-bd91-c377e3b96891","name":"counter-auth-kk8bt","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-auth-kk8bt\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-mvz6h","name":"counter-auth-kk8bt"}}
{"level":"info","ts":"2026-11-03T15:09:10.474Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"e3162694-33e0-4268-bd91-c377e3b96891"}
{"level":"error","ts":"2026-11-03T15:09:10.474Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"e3162694-33e0-4268-bd91-c377e3b96891","error":"mTLS secrets not ready: certificate ray-head-cert-counter-auth-kk8bt not found: certificates.cert-manager.io \"ray-head-cert-counter-auth-kk8bt\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:09:10.484Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"67eca993-3b47-4fa0-b7b1-897516c1bf23"}
{"level":"info","ts":"2026-11-03T15:09:10.484Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"67eca993-3b47-4fa0-b7b1-897516c1bf23"}
{"level":"info","ts":"2026-11-03T15:09:10.484Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"67eca993-3b47-4fa0-b7b1-897516c1bf23","1 head service found":"counter-auth-kk8bt-head-svc"}
{"level":"info","ts":"2026-11-03T15:09:10.484Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"67eca993-3b47-4fa0-b7b1-897516c1bf23"}
{"level":"info","ts":"2026-11-03T15:09:10.497Z","logger":"raycluster-resource","msg":"default","name":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:09:10.498Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"}
{"level":"info","ts":"2026-11-03T15:09:10.534Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"67eca993-3b47-4fa0-b7b1-897516c1bf23","error":"certificate ray-head-cert-counter-auth-kk8bt not found: certificates.cert-manager.io \"ray-head-cert-counter-auth-kk8bt\" not found"}
{"level":"error","ts":"2026-11-03T15:09:10.534Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"67eca993-3b47-4fa0-b7b1-897516c1bf23","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-counter-auth-kk8bt not found: certificates.cert-manager.io \"ray-head-cert-counter-auth-kk8bt\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:09:10.534Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"67eca993-3b47-4fa0-b7b1-897516c1bf23","error":"mTLS secrets not ready: certificate ray-head-cert-counter-auth-kk8bt not found: certificates.cert-manager.io \"ray-head-cert-counter-auth-kk8bt\" not found"}
{"level":"info","ts":"2026-11-03T15:09:10.534Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"67eca993-3b47-4fa0-b7b1-897516c1bf23","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"counter-auth-kk8bt","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:09:10.534Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"67eca993-3b47-4fa0-b7b1-897516c1bf23","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"counter-auth-kk8bt","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:09:10.534Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"67eca993-3b47-4fa0-b7b1-897516c1bf23","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"counter-auth-kk8bt","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:09:10.534Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"67eca993-3b47-4fa0-b7b1-897516c1bf23"}
{"level":"error","ts":"2026-11-03T15:09:10.534Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"67eca993-3b47-4fa0-b7b1-897516c1bf23","error":"mTLS secrets not ready: certificate ray-head-cert-counter-auth-kk8bt not found: certificates.cert-manager.io \"ray-head-cert-counter-auth-kk8bt\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:09:10.554Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"b33f788b-2ba6-4fd3-a8c8-88feb5e7c04a"}
{"level":"info","ts":"2026-11-03T15:09:10.554Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"b33f788b-2ba6-4fd3-a8c8-88feb5e7c04a"}
{"level":"info","ts":"2026-11-03T15:09:10.554Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"b33f788b-2ba6-4fd3-a8c8-88feb5e7c04a","1 head service found":"counter-auth-kk8bt-head-svc"}
{"level":"info","ts":"2026-11-03T15:09:10.554Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"b33f788b-2ba6-4fd3-a8c8-88feb5e7c04a"}
{"level":"info","ts":"2026-11-03T15:09:10.556Z","logger":"raycluster-resource","msg":"validate update","name":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:09:10.604Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"b33f788b-2ba6-4fd3-a8c8-88feb5e7c04a","error":"certificate ray-head-cert-counter-auth-kk8bt not found: certificates.cert-manager.io \"ray-head-cert-counter-auth-kk8bt\" not found"}
{"level":"error","ts":"2026-11-03T15:09:10.604Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"b33f788b-2ba6-4fd3-a8c8-88feb5e7c04a","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-counter-auth-kk8bt not found: certificates.cert-manager.io \"ray-head-cert-counter-auth-kk8bt\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:09:10.604Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"b33f788b-2ba6-4fd3-a8c8-88feb5e7c04a","error":"mTLS secrets not ready: certificate ray-head-cert-counter-auth-kk8bt not found: certificates.cert-manager.io \"ray-head-cert-counter-auth-kk8bt\" not found"}
{"level":"info","ts":"2026-11-03T15:09:10.604Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"b33f788b-2ba6-4fd3-a8c8-88feb5e7c04a","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"counter-auth-kk8bt","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:09:10.604Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"b33f788b-2ba6-4fd3-a8c8-88feb5e7c04a","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"counter-auth-kk8bt","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:09:10.604Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"b33f788b-2ba6-4fd3-a8c8-88feb5e7c04a","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"counter-auth-kk8bt","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:09:10.604Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"b33f788b-2ba6-4fd3-a8c8-88feb5e7c04a"}
{"level":"error","ts":"2026-11-03T15:09:10.604Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"b33f788b-2ba6-4fd3-a8c8-88feb5e7c04a","error":"mTLS secrets not ready: certificate ray-head-cert-counter-auth-kk8bt not found: certificates.cert-manager.io \"ray-head-cert-counter-auth-kk8bt\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:09:10.611Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"4521c6bb-b266-4fbf-9589-e885a74f9bd1","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:09:10.611Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"4521c6bb-b266-4fbf-9589-e885a74f9bd1","name":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:09:10.611Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"4521c6bb-b266-4fbf-9589-e885a74f9bd1","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:09:10.611Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"4521c6bb-b266-4fbf-9589-e885a74f9bd1"}
{"level":"info","ts":"2026-11-03T15:09:10.611Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"4521c6bb-b266-4fbf-9589-e885a74f9bd1","RayCluster":"counter-auth-kk8bt","State":""}
{"level":"info","ts":"2026-11-03T15:09:10.611Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"4521c6bb-b266-4fbf-9589-e885a74f9bd1","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:09:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:09:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:09:10.645Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"c3e11893-8c9a-4121-8a0e-b99e8c9b5007"}
{"level":"info","ts":"2026-11-03T15:09:10.645Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"c3e11893-8c9a-4121-8a0e-b99e8c9b5007"}
{"level":"info","ts":"2026-11-03T15:09:10.645Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"c3e11893-8c9a-4121-8a0e-b99e8c9b5007","1 head service found":"counter-auth-kk8bt-head-svc"}
{"level":"info","ts":"2026-11-03T15:09:10.645Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"c3e11893-8c9a-4121-8a0e-b99e8c9b5007"}
{"level":"info","ts":"2026-11-03T15:09:10.662Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-auth-kk8bt","reconcileID":"9d1700f5-8f1d-4aaa-8c5a-a9ea01b51ee7","name":"counter-auth-kk8bt-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-11-03T15:09:10.695Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"c3e11893-8c9a-4121-8a0e-b99e8c9b5007","error":"certificate ray-head-cert-counter-auth-kk8bt not found: certificates.cert-manager.io \"ray-head-cert-counter-auth-kk8bt\" not found"}
{"level":"error","ts":"2026-11-03T15:09:10.695Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"c3e11893-8c9a-4121-8a0e-b99e8c9b5007","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-counter-auth-kk8bt not found: certificates.cert-manager.io \"ray-head-cert-counter-auth-kk8bt\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:09:10.695Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"c3e11893-8c9a-4121-8a0e-b99e8c9b5007","error":"mTLS secrets not ready: certificate ray-head-cert-counter-auth-kk8bt not found: certificates.cert-manager.io \"ray-head-cert-counter-auth-kk8bt\" not found"}
{"level":"info","ts":"2026-11-03T15:09:10.695Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"c3e11893-8c9a-4121-8a0e-b99e8c9b5007","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"counter-auth-kk8bt","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:09:10.695Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"c3e11893-8c9a-4121-8a0e-b99e8c9b5007","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"counter-auth-kk8bt","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:09:10.695Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"c3e11893-8c9a-4121-8a0e-b99e8c9b5007","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"counter-auth-kk8bt","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:09:10.695Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"c3e11893-8c9a-4121-8a0e-b99e8c9b5007"}
{"level":"error","ts":"2026-11-03T15:09:10.695Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"c3e11893-8c9a-4121-8a0e-b99e8c9b5007","error":"mTLS secrets not ready: certificate ray-head-cert-counter-auth-kk8bt not found: certificates.cert-manager.io \"ray-head-cert-counter-auth-kk8bt\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:09:10.714Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-auth-kk8bt","reconcileID":"9d1700f5-8f1d-4aaa-8c5a-a9ea01b51ee7","name":"kuberay-gateway-access","namespace":"test-ns-mvz6h","operation":"created"}
{"level":"info","ts":"2026-11-03T15:09:10.770Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-auth-kk8bt","reconcileID":"9d1700f5-8f1d-4aaa-8c5a-a9ea01b51ee7","name":"test-ns-mvz6h-counter-auth-kk8bt","operation":"created"}
{"level":"info","ts":"2026-11-03T15:09:10.776Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"c8a3e23d-6d1f-439b-ba57-2c0f9d8a6443"}
{"level":"info","ts":"2026-11-03T15:09:10.776Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"c8a3e23d-6d1f-439b-ba57-2c0f9d8a6443"}
{"level":"info","ts":"2026-11-03T15:09:10.776Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"c8a3e23d-6d1f-439b-ba57-2c0f9d8a6443","1 head service found":"counter-auth-kk8bt-head-svc"}
{"level":"info","ts":"2026-11-03T15:09:10.776Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"c8a3e23d-6d1f-439b-ba57-2c0f9d8a6443"}
{"level":"info","ts":"2026-11-03T15:09:10.822Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-auth-kk8bt","reconcileID":"9d1700f5-8f1d-4aaa-8c5a-a9ea01b51ee7","name":"kube-rbac-proxy-config-counter-auth-kk8bt","operation":"created"}
{"level":"info","ts":"2026-11-03T15:09:10.825Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"c8a3e23d-6d1f-439b-ba57-2c0f9d8a6443","error":"certificate ray-head-cert-counter-auth-kk8bt not found: certificates.cert-manager.io \"ray-head-cert-counter-auth-kk8bt\" not found"}
{"level":"error","ts":"2026-11-03T15:09:10.825Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"c8a3e23d-6d1f-439b-ba57-2c0f9d8a6443","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-counter-auth-kk8bt not found: certificates.cert-manager.io \"ray-head-cert-counter-auth-kk8bt\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:09:10.825Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"c8a3e23d-6d1f-439b-ba57-2c0f9d8a6443","error":"mTLS secrets not ready: certificate ray-head-cert-counter-auth-kk8bt not found: certificates.cert-manager.io \"ray-head-cert-counter-auth-kk8bt\" not found"}
{"level":"info","ts":"2026-11-03T15:09:10.825Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"c8a3e23d-6d1f-439b-ba57-2c0f9d8a6443","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"counter-auth-kk8bt","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:09:10.825Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"c8a3e23d-6d1f-439b-ba57-2c0f9d8a6443","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"counter-auth-kk8bt","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:09:10.825Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"c8a3e23d-6d1f-439b-ba57-2c0f9d8a6443","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"counter-auth-kk8bt","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:09:10.825Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"c8a3e23d-6d1f-439b-ba57-2c0f9d8a6443"}
{"level":"error","ts":"2026-11-03T15:09:10.825Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"c8a3e23d-6d1f-439b-ba57-2c0f9d8a6443","error":"mTLS secrets not ready: certificate ray-head-cert-counter-auth-kk8bt not found: certificates.cert-manager.io \"ray-head-cert-counter-auth-kk8bt\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-03T15:09:10.877Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-auth-kk8bt","reconcileID":"9d1700f5-8f1d-4aaa-8c5a-a9ea01b51ee7","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"counter-auth-kk8bt\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:167\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:09:10.877Z","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-auth-kk8bt","reconcileID":"9d1700f5-8f1d-4aaa-8c5a-a9ea01b51ee7"}
{"level":"error","ts":"2026-11-03T15:09:10.877Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-auth-kk8bt","reconcileID":"9d1700f5-8f1d-4aaa-8c5a-a9ea01b51ee7","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"counter-auth-kk8bt\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:09:10.877Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-auth-kk8bt","reconcileID":"4307882f-a46a-4bbf-956c-5dec8942ea86","namespacedName":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:09:10.877Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-auth-kk8bt","reconcileID":"4307882f-a46a-4bbf-956c-5dec8942ea86","mode":"IntegratedOAuth","cluster":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:09:10.877Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-auth-kk8bt","reconcileID":"4307882f-a46a-4bbf-956c-5dec8942ea86","cluster":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:09:10.938Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-auth-kk8bt","reconcileID":"4307882f-a46a-4bbf-956c-5dec8942ea86","cluster":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:09:10.938Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"3e20c588-9f39-4e28-91cd-411b94c4dbed","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:09:10.938Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"3e20c588-9f39-4e28-91cd-411b94c4dbed","name":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:09:10.938Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"3e20c588-9f39-4e28-91cd-411b94c4dbed","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:09:10.938Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"3e20c588-9f39-4e28-91cd-411b94c4dbed"}
{"level":"info","ts":"2026-11-03T15:09:10.938Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"3e20c588-9f39-4e28-91cd-411b94c4dbed","RayCluster":"counter-auth-kk8bt","State":""}
{"level":"info","ts":"2026-11-03T15:09:10.938Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"3e20c588-9f39-4e28-91cd-411b94c4dbed","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:09:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:09:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:09:10.938Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"3e20c588-9f39-4e28-91cd-411b94c4dbed","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:09:10.986Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"baa8d945-92e2-4667-8ede-d53042daac80"}
{"level":"info","ts":"2026-11-03T15:09:10.986Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"baa8d945-92e2-4667-8ede-d53042daac80"}
{"level":"info","ts":"2026-11-03T15:09:10.986Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"baa8d945-92e2-4667-8ede-d53042daac80","1 head service found":"counter-auth-kk8bt-head-svc"}
{"level":"info","ts":"2026-11-03T15:09:10.986Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"baa8d945-92e2-4667-8ede-d53042daac80"}
{"level":"info","ts":"2026-11-03T15:09:10.998Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"a86cbe5b-4239-4a50-b695-08c0a6aa61be","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:09:10.998Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"a86cbe5b-4239-4a50-b695-08c0a6aa61be","name":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:09:10.998Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"a86cbe5b-4239-4a50-b695-08c0a6aa61be","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:09:10.998Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"a86cbe5b-4239-4a50-b695-08c0a6aa61be"}
{"level":"info","ts":"2026-11-03T15:09:10.998Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"a86cbe5b-4239-4a50-b695-08c0a6aa61be","RayCluster":"counter-auth-kk8bt","State":""}
{"level":"info","ts":"2026-11-03T15:09:10.998Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"a86cbe5b-4239-4a50-b695-08c0a6aa61be","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:09:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:09:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:09:11.037Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"baa8d945-92e2-4667-8ede-d53042daac80","error":"certificate ray-head-cert-counter-auth-kk8bt is not ready at current generation 1"}
{"level":"error","ts":"2026-11-03T15:09:11.037Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"baa8d945-92e2-4667-8ede-d53042daac80","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-counter-auth-kk8bt is not ready at current generation 1","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:09:11.037Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"baa8d945-92e2-4667-8ede-d53042daac80","error":"mTLS secrets not ready: certificate ray-head-cert-counter-auth-kk8bt is not ready at current generation 1"}
{"level":"info","ts":"2026-11-03T15:09:11.037Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"baa8d945-92e2-4667-8ede-d53042daac80","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"counter-auth-kk8bt","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:09:11.037Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"baa8d945-92e2-4667-8ede-d53042daac80","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"counter-auth-kk8bt","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:09:11.037Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"baa8d945-92e2-4667-8ede-d53042daac80","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"counter-auth-kk8bt","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:09:11.037Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"baa8d945-92e2-4667-8ede-d53042daac80"}
{"level":"error","ts":"2026-11-03T15:09:11.037Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"baa8d945-92e2-4667-8ede-d53042daac80","error":"mTLS secrets not ready: certificate ray-head-cert-counter-auth-kk8bt is not ready at current generation 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:09:11.094Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"c414c77e-036f-4cd5-b77c-69ac2e4dc2a8","certificate":"ray-head-cert-counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:09:11.094Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"c414c77e-036f-4cd5-b77c-69ac2e4dc2a8"}
{"level":"info","ts":"2026-11-03T15:09:11.094Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"3c63a6c9-14cd-4aa2-8d91-3314fbf33209","cluster":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:09:11.358Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"e9c59315-8465-4950-bad6-38c435403c90"}
{"level":"info","ts":"2026-11-03T15:09:11.358Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"e9c59315-8465-4950-bad6-38c435403c90"}
{"level":"info","ts":"2026-11-03T15:09:11.358Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"e9c59315-8465-4950-bad6-38c435403c90","1 head service found":"counter-auth-kk8bt-head-svc"}
{"level":"info","ts":"2026-11-03T15:09:11.358Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"e9c59315-8465-4950-bad6-38c435403c90"}
{"level":"info","ts":"2026-11-03T15:09:11.408Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"e9c59315-8465-4950-bad6-38c435403c90","error":"certificate ray-head-cert-counter-auth-kk8bt is not ready at current generation 1"}
{"level":"error","ts":"2026-11-03T15:09:11.408Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"e9c59315-8465-4950-bad6-38c435403c90","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-counter-auth-kk8bt is not ready at current generation 1","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:09:11.408Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"e9c59315-8465-4950-bad6-38c435403c90","error":"mTLS secrets not ready: certificate ray-head-cert-counter-auth-kk8bt is not ready at current generation 1"}
{"level":"info","ts":"2026-11-03T15:09:11.408Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"e9c59315-8465-4950-bad6-38c435403c90","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"counter-auth-kk8bt","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:09:11.408Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"e9c59315-8465-4950-bad6-38c435403c90","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"counter-auth-kk8bt","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:09:11.408Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"e9c59315-8465-4950-bad6-38c435403c90","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"counter-auth-kk8bt","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:09:11.409Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"e9c59315-8465-4950-bad6-38c435403c90"}
{"level":"error","ts":"2026-11-03T15:09:11.409Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"e9c59315-8465-4950-bad6-38c435403c90","error":"mTLS secrets not ready: certificate ray-head-cert-counter-auth-kk8bt is not ready at current generation 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:09:11.545Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"3c63a6c9-14cd-4aa2-8d91-3314fbf33209","certificate":"ray-head-cert-counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:09:11.545Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"3c63a6c9-14cd-4aa2-8d91-3314fbf33209"}
{"level":"info","ts":"2026-11-03T15:09:12.049Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"d293c62f-9c5d-42d9-9780-cc5fb98c3787"}
{"level":"info","ts":"2026-11-03T15:09:12.049Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"d293c62f-9c5d-42d9-9780-cc5fb98c3787"}
{"level":"info","ts":"2026-11-03T15:09:12.049Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"d293c62f-9c5d-42d9-9780-cc5fb98c3787","1 head service found":"counter-auth-kk8bt-head-svc"}
{"level":"info","ts":"2026-11-03T15:09:12.049Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"d293c62f-9c5d-42d9-9780-cc5fb98c3787"}
{"level":"info","ts":"2026-11-03T15:09:12.100Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"d293c62f-9c5d-42d9-9780-cc5fb98c3787","error":"certificate ray-head-cert-counter-auth-kk8bt is not ready at current generation 1"}
{"level":"error","ts":"2026-11-03T15:09:12.100Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"d293c62f-9c5d-42d9-9780-cc5fb98c3787","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-counter-auth-kk8bt is not ready at current generation 1","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:09:12.100Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"d293c62f-9c5d-42d9-9780-cc5fb98c3787","error":"mTLS secrets not ready: certificate ray-head-cert-counter-auth-kk8bt is not ready at current generation 1"}
{"level":"info","ts":"2026-11-03T15:09:12.100Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"d293c62f-9c5d-42d9-9780-cc5fb98c3787","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"counter-auth-kk8bt","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:09:12.100Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"d293c62f-9c5d-42d9-9780-cc5fb98c3787","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"counter-auth-kk8bt","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:09:12.100Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"d293c62f-9c5d-42d9-9780-cc5fb98c3787","filter labels":["test-ns-mvz6h",{"ray.io/cluster":"counter-auth-kk8bt","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:09:12.100Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"d293c62f-9c5d-42d9-9780-cc5fb98c3787"}
{"level":"error","ts":"2026-11-03T15:09:12.100Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"d293c62f-9c5d-42d9-9780-cc5fb98c3787","error":"mTLS secrets not ready: certificate ray-head-cert-counter-auth-kk8bt is not ready at current generation 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:09:13.381Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"e45592e9-9825-4942-8c21-3ab0e95da17b"}
{"level":"info","ts":"2026-11-03T15:09:13.381Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"e45592e9-9825-4942-8c21-3ab0e95da17b"}
{"level":"info","ts":"2026-11-03T15:09:13.381Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"e45592e9-9825-4942-8c21-3ab0e95da17b","1 head service found":"counter-auth-kk8bt-head-svc"}
{"level":"info","ts":"2026-11-03T15:09:13.381Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"e45592e9-9825-4942-8c21-3ab0e95da17b"}
{"level":"info","ts":"2026-11-03T15:09:13.483Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"e45592e9-9825-4942-8c21-3ab0e95da17b","mode":"IntegratedOAuth","cluster":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:09:13.483Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"e45592e9-9825-4942-8c21-3ab0e95da17b","cluster":"counter-auth-kk8bt","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-11-03T15:09:13.483Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"e45592e9-9825-4942-8c21-3ab0e95da17b","cluster":"counter-auth-kk8bt","authType":"IntegratedOAuth","serviceAccount":"counter-auth-kk8bt-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-11-03T15:09:13.483Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"e45592e9-9825-4942-8c21-3ab0e95da17b","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"counter-auth-kk8bt","ray.io/group":"headgroup","ray.io/identifier":"counter-auth-kk8bt-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-11-03T15:09:13.483Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"e45592e9-9825-4942-8c21-3ab0e95da17b","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-11-03T15:09:13.483Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"e45592e9-9825-4942-8c21-3ab0e95da17b","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-11-03T15:09:13.483Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"e45592e9-9825-4942-8c21-3ab0e95da17b","rayNodeType":"head","generatedCmd":"ulimit -n ${RAY_START_ULIMIT_OPEN_FILES:-65536}; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-11-03T15:09:13.483Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"e45592e9-9825-4942-8c21-3ab0e95da17b","enabled":true}
{"level":"info","ts":"2026-11-03T15:09:13.555Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"e45592e9-9825-4942-8c21-3ab0e95da17b","name":"counter-auth-kk8bt-head-27gn8"}
{"level":"info","ts":"2026-11-03T15:09:13.555Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"e45592e9-9825-4942-8c21-3ab0e95da17b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:09:13.555Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"e45592e9-9825-4942-8c21-3ab0e95da17b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:09:13.555Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"e45592e9-9825-4942-8c21-3ab0e95da17b","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-11-03T15:09:13.555Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"e45592e9-9825-4942-8c21-3ab0e95da17b","group":"small-group","indices":{}}
{"level":"info","ts":"2026-11-03T15:09:13.555Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"e45592e9-9825-4942-8c21-3ab0e95da17b","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-11-03T15:09:13.656Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"e45592e9-9825-4942-8c21-3ab0e95da17b","creating worker for group":"small-group","index":0,"total":1,"replicaIndex":0}
{"level":"info","ts":"2026-11-03T15:09:13.656Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"e45592e9-9825-4942-8c21-3ab0e95da17b","nodeType":"worker","rayStartParams":{"address":"counter-auth-kk8bt-head-svc.test-ns-mvz6h.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-11-03T15:09:13.656Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"e45592e9-9825-4942-8c21-3ab0e95da17b","rayStartCmd":"ray start  --address=counter-auth-kk8bt-head-svc.test-ns-mvz6h.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-11-03T15:09:13.656Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"e45592e9-9825-4942-8c21-3ab0e95da17b","rayNodeType":"worker","generatedCmd":"ulimit -n ${RAY_START_ULIMIT_OPEN_FILES:-65536}; ray start  --address=counter-auth-kk8bt-head-svc.test-ns-mvz6h.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-11-03T15:09:13.656Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"e45592e9-9825-4942-8c21-3ab0e95da17b","enabled":true}
{"level":"info","ts":"2026-11-03T15:09:13.718Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"e45592e9-9825-4942-8c21-3ab0e95da17b","name":"counter-auth-kk8bt-small-group-worker-k56zq"}
{"level":"info","ts":"2026-11-03T15:09:13.718Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"e45592e9-9825-4942-8c21-3ab0e95da17b","podName":"counter-auth-kk8bt-head-27gn8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:09:13.718Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"e45592e9-9825-4942-8c21-3ab0e95da17b","name":"counter-auth-kk8bt","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:09:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:09:13Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:09:13.780Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"59d48385-8dc0-4367-a32c-e12594acb322","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:09:13.780Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"59d48385-8dc0-4367-a32c-e12594acb322","name":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:09:13.780Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"59d48385-8dc0-4367-a32c-e12594acb322","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:09:13.780Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"a8c3c019-905e-467e-a122-d80f0ee6f79c","cluster":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:09:13.781Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"59d48385-8dc0-4367-a32c-e12594acb322"}
{"level":"info","ts":"2026-11-03T15:09:13.781Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"59d48385-8dc0-4367-a32c-e12594acb322","RayCluster":"counter-auth-kk8bt","State":""}
{"level":"info","ts":"2026-11-03T15:09:13.781Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"59d48385-8dc0-4367-a32c-e12594acb322","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:09:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:09:13Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:09:13.781Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"59d48385-8dc0-4367-a32c-e12594acb322","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:09:13.781Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"f32cb31f-9d71-4c97-8ec7-e7fa698bd660"}
{"level":"info","ts":"2026-11-03T15:09:13.781Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"f32cb31f-9d71-4c97-8ec7-e7fa698bd660"}
{"level":"info","ts":"2026-11-03T15:09:13.781Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"f32cb31f-9d71-4c97-8ec7-e7fa698bd660","1 head service found":"counter-auth-kk8bt-head-svc"}
{"level":"info","ts":"2026-11-03T15:09:13.781Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"f32cb31f-9d71-4c97-8ec7-e7fa698bd660","Found 1 head Pod":"counter-auth-kk8bt-head-27gn8","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:09:13.781Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"f32cb31f-9d71-4c97-8ec7-e7fa698bd660","head Pod":"counter-auth-kk8bt-head-27gn8","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-kk8bt-head-27gn8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:09:13.781Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"f32cb31f-9d71-4c97-8ec7-e7fa698bd660","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:09:13.781Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"f32cb31f-9d71-4c97-8ec7-e7fa698bd660","worker Pod":"counter-auth-kk8bt-small-group-worker-k56zq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-kk8bt-small-group-worker-k56zq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:09:13.781Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"f32cb31f-9d71-4c97-8ec7-e7fa698bd660","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:09:13.781Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"f32cb31f-9d71-4c97-8ec7-e7fa698bd660","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:09:13.781Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"f32cb31f-9d71-4c97-8ec7-e7fa698bd660","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:09:13.781Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"f32cb31f-9d71-4c97-8ec7-e7fa698bd660","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:09:13.781Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"f32cb31f-9d71-4c97-8ec7-e7fa698bd660","podName":"counter-auth-kk8bt-head-27gn8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:09:13.781Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"f32cb31f-9d71-4c97-8ec7-e7fa698bd660","podName":"counter-auth-kk8bt-head-27gn8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:09:13.781Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"f32cb31f-9d71-4c97-8ec7-e7fa698bd660","name":"counter-auth-kk8bt","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:09:13Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:09:13Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:09:13.842Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"7e23a1fc-a034-4e80-beb2-304ac87ab8ee","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:09:13.842Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"7e23a1fc-a034-4e80-beb2-304ac87ab8ee","name":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:09:13.842Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"7e23a1fc-a034-4e80-beb2-304ac87ab8ee","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:09:13.842Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"7e23a1fc-a034-4e80-beb2-304ac87ab8ee"}
{"level":"info","ts":"2026-11-03T15:09:13.842Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"7e23a1fc-a034-4e80-beb2-304ac87ab8ee","RayCluster":"counter-auth-kk8bt","State":""}
{"level":"info","ts":"2026-11-03T15:09:13.842Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"7e23a1fc-a034-4e80-beb2-304ac87ab8ee","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:09:13Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:09:13Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:09:13.846Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"6c84a964-53eb-4348-ae7d-d2f3fb69631a"}
{"level":"info","ts":"2026-11-03T15:09:13.846Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"6c84a964-53eb-4348-ae7d-d2f3fb69631a"}
{"level":"info","ts":"2026-11-03T15:09:13.846Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"6c84a964-53eb-4348-ae7d-d2f3fb69631a","1 head service found":"counter-auth-kk8bt-head-svc"}
{"level":"info","ts":"2026-11-03T15:09:13.846Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"6c84a964-53eb-4348-ae7d-d2f3fb69631a","Found 1 head Pod":"counter-auth-kk8bt-head-27gn8","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:09:13.846Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"6c84a964-53eb-4348-ae7d-d2f3fb69631a","head Pod":"counter-auth-kk8bt-head-27gn8","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-kk8bt-head-27gn8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:09:13.846Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"6c84a964-53eb-4348-ae7d-d2f3fb69631a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:09:13.846Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"6c84a964-53eb-4348-ae7d-d2f3fb69631a","worker Pod":"counter-auth-kk8bt-small-group-worker-k56zq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-kk8bt-small-group-worker-k56zq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:09:13.846Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"6c84a964-53eb-4348-ae7d-d2f3fb69631a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:09:13.846Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"6c84a964-53eb-4348-ae7d-d2f3fb69631a","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:09:13.846Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"6c84a964-53eb-4348-ae7d-d2f3fb69631a","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:09:13.846Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"6c84a964-53eb-4348-ae7d-d2f3fb69631a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:09:13.846Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"6c84a964-53eb-4348-ae7d-d2f3fb69631a","podName":"counter-auth-kk8bt-head-27gn8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:09:13.846Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"6c84a964-53eb-4348-ae7d-d2f3fb69631a","podName":"counter-auth-kk8bt-head-27gn8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:09:13.846Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"6c84a964-53eb-4348-ae7d-d2f3fb69631a","name":"counter-auth-kk8bt","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:09:13Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:09:13Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:09:13.846Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"f5fb8613-5691-433c-97ea-1d4aa80d0a4b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:09:13.846Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"f5fb8613-5691-433c-97ea-1d4aa80d0a4b","name":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:09:13.846Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"f5fb8613-5691-433c-97ea-1d4aa80d0a4b","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:09:13.847Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"f5fb8613-5691-433c-97ea-1d4aa80d0a4b"}
{"level":"info","ts":"2026-11-03T15:09:13.847Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"f5fb8613-5691-433c-97ea-1d4aa80d0a4b","RayCluster":"counter-auth-kk8bt","State":""}
{"level":"info","ts":"2026-11-03T15:09:13.847Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"f5fb8613-5691-433c-97ea-1d4aa80d0a4b","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:09:13Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:09:13Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:09:13.847Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"f5fb8613-5691-433c-97ea-1d4aa80d0a4b","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:09:13.901Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"6c84a964-53eb-4348-ae7d-d2f3fb69631a","name":"counter-auth-kk8bt","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-auth-kk8bt\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-mvz6h","name":"counter-auth-kk8bt"}}
{"level":"info","ts":"2026-11-03T15:09:13.901Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"6c84a964-53eb-4348-ae7d-d2f3fb69631a"}
{"level":"error","ts":"2026-11-03T15:09:13.901Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"6c84a964-53eb-4348-ae7d-d2f3fb69631a","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-auth-kk8bt\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:09:13.901Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"86f0ffbe-c109-4fec-8735-c86f033f2b86"}
{"level":"info","ts":"2026-11-03T15:09:13.901Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"86f0ffbe-c109-4fec-8735-c86f033f2b86"}
{"level":"info","ts":"2026-11-03T15:09:13.901Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"86f0ffbe-c109-4fec-8735-c86f033f2b86","1 head service found":"counter-auth-kk8bt-head-svc"}
{"level":"info","ts":"2026-11-03T15:09:13.901Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"86f0ffbe-c109-4fec-8735-c86f033f2b86","Found 1 head Pod":"counter-auth-kk8bt-head-27gn8","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:09:13.901Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"86f0ffbe-c109-4fec-8735-c86f033f2b86","head Pod":"counter-auth-kk8bt-head-27gn8","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-kk8bt-head-27gn8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:09:13.901Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"86f0ffbe-c109-4fec-8735-c86f033f2b86","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:09:13.901Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"86f0ffbe-c109-4fec-8735-c86f033f2b86","worker Pod":"counter-auth-kk8bt-small-group-worker-k56zq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-kk8bt-small-group-worker-k56zq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:09:13.901Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"86f0ffbe-c109-4fec-8735-c86f033f2b86","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:09:13.901Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"86f0ffbe-c109-4fec-8735-c86f033f2b86","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:09:13.901Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"86f0ffbe-c109-4fec-8735-c86f033f2b86","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:09:13.901Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"86f0ffbe-c109-4fec-8735-c86f033f2b86","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:09:13.901Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"86f0ffbe-c109-4fec-8735-c86f033f2b86","podName":"counter-auth-kk8bt-head-27gn8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:09:13.901Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"86f0ffbe-c109-4fec-8735-c86f033f2b86","podName":"counter-auth-kk8bt-head-27gn8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:09:13.902Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"86f0ffbe-c109-4fec-8735-c86f033f2b86","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:09:13.902Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"86f0ffbe-c109-4fec-8735-c86f033f2b86","seconds":300}
{"level":"info","ts":"2026-11-03T15:09:13.907Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"d7d4cedb-8eea-416b-ac96-dacaf10c4dbf","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:09:13.907Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"d7d4cedb-8eea-416b-ac96-dacaf10c4dbf","name":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:09:13.907Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"d7d4cedb-8eea-416b-ac96-dacaf10c4dbf","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:09:13.908Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"d7d4cedb-8eea-416b-ac96-dacaf10c4dbf"}
{"level":"info","ts":"2026-11-03T15:09:13.908Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"d7d4cedb-8eea-416b-ac96-dacaf10c4dbf","RayCluster":"counter-auth-kk8bt","State":""}
{"level":"info","ts":"2026-11-03T15:09:13.908Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"d7d4cedb-8eea-416b-ac96-dacaf10c4dbf","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:09:13Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:09:13Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:09:14.350Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"5148fd19-1c93-4608-8016-90f034761199"}
{"level":"info","ts":"2026-11-03T15:09:14.350Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"5148fd19-1c93-4608-8016-90f034761199"}
{"level":"info","ts":"2026-11-03T15:09:14.350Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"5148fd19-1c93-4608-8016-90f034761199","1 head service found":"counter-auth-kk8bt-head-svc"}
{"level":"info","ts":"2026-11-03T15:09:14.350Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"5148fd19-1c93-4608-8016-90f034761199","Found 1 head Pod":"counter-auth-kk8bt-head-27gn8","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:09:14.350Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"5148fd19-1c93-4608-8016-90f034761199","head Pod":"counter-auth-kk8bt-head-27gn8","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-kk8bt-head-27gn8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:09:14.350Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"5148fd19-1c93-4608-8016-90f034761199","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:09:14.350Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"5148fd19-1c93-4608-8016-90f034761199","worker Pod":"counter-auth-kk8bt-small-group-worker-k56zq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-kk8bt-small-group-worker-k56zq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:09:14.350Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"5148fd19-1c93-4608-8016-90f034761199","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:09:14.350Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"5148fd19-1c93-4608-8016-90f034761199","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:09:14.350Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"5148fd19-1c93-4608-8016-90f034761199","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:09:14.350Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"5148fd19-1c93-4608-8016-90f034761199","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:09:14.350Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"5148fd19-1c93-4608-8016-90f034761199","podName":"counter-auth-kk8bt-head-27gn8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:09:14.350Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"5148fd19-1c93-4608-8016-90f034761199","podName":"counter-auth-kk8bt-head-27gn8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:09:14.350Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"5148fd19-1c93-4608-8016-90f034761199","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:09:14.350Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"5148fd19-1c93-4608-8016-90f034761199","seconds":300}
{"level":"info","ts":"2026-11-03T15:09:14.521Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"b09e20b6-dd23-4c27-8859-b0a21fcccedf"}
{"level":"info","ts":"2026-11-03T15:09:14.521Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"b09e20b6-dd23-4c27-8859-b0a21fcccedf"}
{"level":"info","ts":"2026-11-03T15:09:14.521Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"b09e20b6-dd23-4c27-8859-b0a21fcccedf","1 head service found":"counter-auth-kk8bt-head-svc"}
{"level":"info","ts":"2026-11-03T15:09:14.521Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"b09e20b6-dd23-4c27-8859-b0a21fcccedf","Found 1 head Pod":"counter-auth-kk8bt-head-27gn8","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:09:14.521Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"b09e20b6-dd23-4c27-8859-b0a21fcccedf","head Pod":"counter-auth-kk8bt-head-27gn8","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-kk8bt-head-27gn8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:09:14.521Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"b09e20b6-dd23-4c27-8859-b0a21fcccedf","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:09:14.521Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"b09e20b6-dd23-4c27-8859-b0a21fcccedf","worker Pod":"counter-auth-kk8bt-small-group-worker-k56zq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-kk8bt-small-group-worker-k56zq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:09:14.521Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"b09e20b6-dd23-4c27-8859-b0a21fcccedf","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:09:14.521Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"b09e20b6-dd23-4c27-8859-b0a21fcccedf","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:09:14.521Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"b09e20b6-dd23-4c27-8859-b0a21fcccedf","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:09:14.521Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"b09e20b6-dd23-4c27-8859-b0a21fcccedf","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:09:14.521Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"b09e20b6-dd23-4c27-8859-b0a21fcccedf","podName":"counter-auth-kk8bt-small-group-worker-k56zq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:09:14.521Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"b09e20b6-dd23-4c27-8859-b0a21fcccedf","podName":"counter-auth-kk8bt-small-group-worker-k56zq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:09:14.521Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"b09e20b6-dd23-4c27-8859-b0a21fcccedf","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:09:14.521Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"b09e20b6-dd23-4c27-8859-b0a21fcccedf","seconds":300}
{"level":"info","ts":"2026-11-03T15:09:14.599Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"fe0bf7da-0730-4727-8d9d-29eaa9f4ff01","cluster":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:09:15.201Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"b5a14082-67f3-4b8a-a5d9-17ddd8ec91f8"}
{"level":"info","ts":"2026-11-03T15:09:15.201Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"b5a14082-67f3-4b8a-a5d9-17ddd8ec91f8"}
{"level":"info","ts":"2026-11-03T15:09:15.201Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"b5a14082-67f3-4b8a-a5d9-17ddd8ec91f8","1 head service found":"counter-auth-kk8bt-head-svc"}
{"level":"info","ts":"2026-11-03T15:09:15.201Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"b5a14082-67f3-4b8a-a5d9-17ddd8ec91f8","Found 1 head Pod":"counter-auth-kk8bt-head-27gn8","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:09:15.201Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"b5a14082-67f3-4b8a-a5d9-17ddd8ec91f8","head Pod":"counter-auth-kk8bt-head-27gn8","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-kk8bt-head-27gn8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:09:15.201Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"b5a14082-67f3-4b8a-a5d9-17ddd8ec91f8","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:09:15.201Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"b5a14082-67f3-4b8a-a5d9-17ddd8ec91f8","worker Pod":"counter-auth-kk8bt-small-group-worker-k56zq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-kk8bt-small-group-worker-k56zq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:09:15.201Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"b5a14082-67f3-4b8a-a5d9-17ddd8ec91f8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:09:15.201Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"b5a14082-67f3-4b8a-a5d9-17ddd8ec91f8","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:09:15.201Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"b5a14082-67f3-4b8a-a5d9-17ddd8ec91f8","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:09:15.201Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"b5a14082-67f3-4b8a-a5d9-17ddd8ec91f8","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:09:15.201Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"b5a14082-67f3-4b8a-a5d9-17ddd8ec91f8","podName":"counter-auth-kk8bt-head-27gn8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:09:15.201Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"b5a14082-67f3-4b8a-a5d9-17ddd8ec91f8","podName":"counter-auth-kk8bt-head-27gn8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:09:15.201Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"b5a14082-67f3-4b8a-a5d9-17ddd8ec91f8","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:09:15.201Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"b5a14082-67f3-4b8a-a5d9-17ddd8ec91f8","seconds":300}
{"level":"info","ts":"2026-11-03T15:09:15.452Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"edd9da90-c362-48b7-9da7-4ba5eb207d30"}
{"level":"info","ts":"2026-11-03T15:09:15.452Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"edd9da90-c362-48b7-9da7-4ba5eb207d30"}
{"level":"info","ts":"2026-11-03T15:09:15.452Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"edd9da90-c362-48b7-9da7-4ba5eb207d30","1 head service found":"counter-auth-kk8bt-head-svc"}
{"level":"info","ts":"2026-11-03T15:09:15.452Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"edd9da90-c362-48b7-9da7-4ba5eb207d30","Found 1 head Pod":"counter-auth-kk8bt-head-27gn8","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:09:15.452Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"edd9da90-c362-48b7-9da7-4ba5eb207d30","head Pod":"counter-auth-kk8bt-head-27gn8","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-kk8bt-head-27gn8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:09:15.452Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"edd9da90-c362-48b7-9da7-4ba5eb207d30","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:09:15.452Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"edd9da90-c362-48b7-9da7-4ba5eb207d30","worker Pod":"counter-auth-kk8bt-small-group-worker-k56zq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-kk8bt-small-group-worker-k56zq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:09:15.452Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"edd9da90-c362-48b7-9da7-4ba5eb207d30","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:09:15.452Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"edd9da90-c362-48b7-9da7-4ba5eb207d30","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:09:15.452Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"edd9da90-c362-48b7-9da7-4ba5eb207d30","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:09:15.452Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"edd9da90-c362-48b7-9da7-4ba5eb207d30","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:09:15.452Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"edd9da90-c362-48b7-9da7-4ba5eb207d30","podName":"counter-auth-kk8bt-head-27gn8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:09:15.452Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"edd9da90-c362-48b7-9da7-4ba5eb207d30","podName":"counter-auth-kk8bt-head-27gn8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:09:15.452Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"edd9da90-c362-48b7-9da7-4ba5eb207d30","name":"counter-auth-kk8bt","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:09:13Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:09:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:09:15.513Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"0a6e3641-a601-4c72-aa5d-2190126ff48d","cluster":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:09:15.513Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"2cd2fb26-03b8-4ead-a3cd-e1525ff1f893","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:09:15.514Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"2cd2fb26-03b8-4ead-a3cd-e1525ff1f893","name":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:09:15.514Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"2cd2fb26-03b8-4ead-a3cd-e1525ff1f893","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:09:15.514Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"2cd2fb26-03b8-4ead-a3cd-e1525ff1f893"}
{"level":"info","ts":"2026-11-03T15:09:15.514Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"2cd2fb26-03b8-4ead-a3cd-e1525ff1f893","RayCluster":"counter-auth-kk8bt","State":""}
{"level":"info","ts":"2026-11-03T15:09:15.514Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"2cd2fb26-03b8-4ead-a3cd-e1525ff1f893","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:09:13Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:09:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:09:15.514Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"2cd2fb26-03b8-4ead-a3cd-e1525ff1f893","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:09:15.574Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"adf69500-14f9-46c7-9032-4904339cc41f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:09:15.574Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"adf69500-14f9-46c7-9032-4904339cc41f","name":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:09:15.574Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"adf69500-14f9-46c7-9032-4904339cc41f","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:09:15.574Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"adf69500-14f9-46c7-9032-4904339cc41f"}
{"level":"info","ts":"2026-11-03T15:09:15.574Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"adf69500-14f9-46c7-9032-4904339cc41f","RayCluster":"counter-auth-kk8bt","State":""}
{"level":"info","ts":"2026-11-03T15:09:15.574Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"adf69500-14f9-46c7-9032-4904339cc41f","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:09:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:09:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:09:15.814Z","logger":"controllers.RayClusterMTLS","msg":"Updating head certificate","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"0a6e3641-a601-4c72-aa5d-2190126ff48d","certificate":"ray-head-cert-counter-auth-kk8bt","ipAddresses":["10.132.0.34","127.0.0.1"]}
{"level":"info","ts":"2026-11-03T15:09:16.068Z","logger":"controllers.RayClusterMTLS","msg":"Updating worker certificate","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"0a6e3641-a601-4c72-aa5d-2190126ff48d","certificate":"ray-worker-cert-counter-auth-kk8bt","ipAddresses":["10.134.0.30","127.0.0.1"]}
{"level":"info","ts":"2026-11-03T15:09:16.324Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"0a6e3641-a601-4c72-aa5d-2190126ff48d","certificate":"ray-head-cert-counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:09:16.324Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"0a6e3641-a601-4c72-aa5d-2190126ff48d"}
{"level":"info","ts":"2026-11-03T15:09:17.514Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"3421f58c-c030-48e4-abbf-95eac2efa67b"}
{"level":"info","ts":"2026-11-03T15:09:17.514Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"3421f58c-c030-48e4-abbf-95eac2efa67b"}
{"level":"info","ts":"2026-11-03T15:09:17.514Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"3421f58c-c030-48e4-abbf-95eac2efa67b","1 head service found":"counter-auth-kk8bt-head-svc"}
{"level":"info","ts":"2026-11-03T15:09:17.515Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"3421f58c-c030-48e4-abbf-95eac2efa67b","Found 1 head Pod":"counter-auth-kk8bt-head-27gn8","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:09:17.515Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"3421f58c-c030-48e4-abbf-95eac2efa67b","head Pod":"counter-auth-kk8bt-head-27gn8","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-kk8bt-head-27gn8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:09:17.515Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"3421f58c-c030-48e4-abbf-95eac2efa67b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:09:17.515Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"3421f58c-c030-48e4-abbf-95eac2efa67b","worker Pod":"counter-auth-kk8bt-small-group-worker-k56zq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-kk8bt-small-group-worker-k56zq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:09:17.515Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"3421f58c-c030-48e4-abbf-95eac2efa67b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:09:17.515Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"3421f58c-c030-48e4-abbf-95eac2efa67b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:09:17.515Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"3421f58c-c030-48e4-abbf-95eac2efa67b","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:09:17.515Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"3421f58c-c030-48e4-abbf-95eac2efa67b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:09:17.515Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"3421f58c-c030-48e4-abbf-95eac2efa67b","podName":"counter-auth-kk8bt-head-27gn8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:09:17.515Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"3421f58c-c030-48e4-abbf-95eac2efa67b","podName":"counter-auth-kk8bt-head-27gn8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:09:17.515Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"3421f58c-c030-48e4-abbf-95eac2efa67b","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:09:17.515Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"3421f58c-c030-48e4-abbf-95eac2efa67b","seconds":300}
{"level":"info","ts":"2026-11-03T15:09:18.575Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"c645211f-0e8a-4e9a-b028-a1a7d66b18a9","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:09:18.575Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"c645211f-0e8a-4e9a-b028-a1a7d66b18a9","name":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:09:18.575Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"c645211f-0e8a-4e9a-b028-a1a7d66b18a9","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:09:18.575Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"c645211f-0e8a-4e9a-b028-a1a7d66b18a9"}
{"level":"info","ts":"2026-11-03T15:09:18.575Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"c645211f-0e8a-4e9a-b028-a1a7d66b18a9","RayCluster":"counter-auth-kk8bt","State":""}
{"level":"info","ts":"2026-11-03T15:09:18.575Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"c645211f-0e8a-4e9a-b028-a1a7d66b18a9","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:09:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:09:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:09:21.576Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"c67116ee-e72e-4775-bcdb-35fa76cf655b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:09:21.576Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"c67116ee-e72e-4775-bcdb-35fa76cf655b","name":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:09:21.576Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"c67116ee-e72e-4775-bcdb-35fa76cf655b","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:09:21.577Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"c67116ee-e72e-4775-bcdb-35fa76cf655b"}
{"level":"info","ts":"2026-11-03T15:09:21.577Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"c67116ee-e72e-4775-bcdb-35fa76cf655b","RayCluster":"counter-auth-kk8bt","State":""}
{"level":"info","ts":"2026-11-03T15:09:21.577Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"c67116ee-e72e-4775-bcdb-35fa76cf655b","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:09:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:09:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:09:24.578Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"a1cf416e-9076-47f0-83fe-7e31fdf26ccf","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:09:24.578Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"a1cf416e-9076-47f0-83fe-7e31fdf26ccf","name":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:09:24.578Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"a1cf416e-9076-47f0-83fe-7e31fdf26ccf","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:09:24.578Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"a1cf416e-9076-47f0-83fe-7e31fdf26ccf"}
{"level":"info","ts":"2026-11-03T15:09:24.578Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"a1cf416e-9076-47f0-83fe-7e31fdf26ccf","RayCluster":"counter-auth-kk8bt","State":""}
{"level":"info","ts":"2026-11-03T15:09:24.578Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"a1cf416e-9076-47f0-83fe-7e31fdf26ccf","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:09:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:09:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:09:27.579Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"afeba74c-280f-4d2d-b4c8-9be6433fe40a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:09:27.579Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"afeba74c-280f-4d2d-b4c8-9be6433fe40a","name":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:09:27.579Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"afeba74c-280f-4d2d-b4c8-9be6433fe40a","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:09:27.579Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"afeba74c-280f-4d2d-b4c8-9be6433fe40a"}
{"level":"info","ts":"2026-11-03T15:09:27.579Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"afeba74c-280f-4d2d-b4c8-9be6433fe40a","RayCluster":"counter-auth-kk8bt","State":""}
{"level":"info","ts":"2026-11-03T15:09:27.579Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"afeba74c-280f-4d2d-b4c8-9be6433fe40a","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:09:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:09:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:09:30.581Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"cbfb7670-f448-4c44-99c3-daa469b0943d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:09:30.581Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"cbfb7670-f448-4c44-99c3-daa469b0943d","name":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:09:30.581Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"cbfb7670-f448-4c44-99c3-daa469b0943d","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:09:30.581Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"cbfb7670-f448-4c44-99c3-daa469b0943d"}
{"level":"info","ts":"2026-11-03T15:09:30.581Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"cbfb7670-f448-4c44-99c3-daa469b0943d","RayCluster":"counter-auth-kk8bt","State":""}
{"level":"info","ts":"2026-11-03T15:09:30.581Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"cbfb7670-f448-4c44-99c3-daa469b0943d","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:09:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:09:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:09:33.582Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"7a2e9e09-1224-409c-b618-ca9ca003c99b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:09:33.582Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"7a2e9e09-1224-409c-b618-ca9ca003c99b","name":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:09:33.582Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"7a2e9e09-1224-409c-b618-ca9ca003c99b","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:09:33.583Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"7a2e9e09-1224-409c-b618-ca9ca003c99b"}
{"level":"info","ts":"2026-11-03T15:09:33.583Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"7a2e9e09-1224-409c-b618-ca9ca003c99b","RayCluster":"counter-auth-kk8bt","State":""}
{"level":"info","ts":"2026-11-03T15:09:33.583Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"7a2e9e09-1224-409c-b618-ca9ca003c99b","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:09:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:09:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:09:36.583Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"208544a1-887d-4718-83c7-1e79d886856d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:09:36.583Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"208544a1-887d-4718-83c7-1e79d886856d","name":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:09:36.583Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"208544a1-887d-4718-83c7-1e79d886856d","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:09:36.584Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"208544a1-887d-4718-83c7-1e79d886856d"}
{"level":"info","ts":"2026-11-03T15:09:36.584Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"208544a1-887d-4718-83c7-1e79d886856d","RayCluster":"counter-auth-kk8bt","State":""}
{"level":"info","ts":"2026-11-03T15:09:36.584Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"208544a1-887d-4718-83c7-1e79d886856d","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:09:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:09:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:09:39.584Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"c9e8c10b-9a36-4c92-99ea-7d8ce2e5bbcd","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:09:39.584Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"c9e8c10b-9a36-4c92-99ea-7d8ce2e5bbcd","name":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:09:39.584Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"c9e8c10b-9a36-4c92-99ea-7d8ce2e5bbcd","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:09:39.585Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"c9e8c10b-9a36-4c92-99ea-7d8ce2e5bbcd"}
{"level":"info","ts":"2026-11-03T15:09:39.585Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"c9e8c10b-9a36-4c92-99ea-7d8ce2e5bbcd","RayCluster":"counter-auth-kk8bt","State":""}
{"level":"info","ts":"2026-11-03T15:09:39.585Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"c9e8c10b-9a36-4c92-99ea-7d8ce2e5bbcd","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:09:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:09:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:09:42.585Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"d9191aac-69ea-4f11-92b2-0db5be6b2a9d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:09:42.585Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"d9191aac-69ea-4f11-92b2-0db5be6b2a9d","name":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:09:42.585Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"d9191aac-69ea-4f11-92b2-0db5be6b2a9d","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:09:42.585Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"d9191aac-69ea-4f11-92b2-0db5be6b2a9d"}
{"level":"info","ts":"2026-11-03T15:09:42.585Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"d9191aac-69ea-4f11-92b2-0db5be6b2a9d","RayCluster":"counter-auth-kk8bt","State":""}
{"level":"info","ts":"2026-11-03T15:09:42.585Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"d9191aac-69ea-4f11-92b2-0db5be6b2a9d","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:09:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:09:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:09:45.586Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"52b55183-016f-48dd-b8e3-ada111c1c009","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:09:45.586Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"52b55183-016f-48dd-b8e3-ada111c1c009","name":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:09:45.586Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"52b55183-016f-48dd-b8e3-ada111c1c009","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:09:45.586Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"52b55183-016f-48dd-b8e3-ada111c1c009"}
{"level":"info","ts":"2026-11-03T15:09:45.586Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"52b55183-016f-48dd-b8e3-ada111c1c009","RayCluster":"counter-auth-kk8bt","State":""}
{"level":"info","ts":"2026-11-03T15:09:45.586Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"52b55183-016f-48dd-b8e3-ada111c1c009","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:09:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:09:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:09:46.325Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"799585d8-9fce-4b6b-9dc2-0d127a98de44","cluster":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:09:48.586Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"4fc69a3d-b1b7-43c1-b971-08b3458974bc","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:09:48.586Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"4fc69a3d-b1b7-43c1-b971-08b3458974bc","name":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:09:48.586Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"4fc69a3d-b1b7-43c1-b971-08b3458974bc","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:09:48.587Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"4fc69a3d-b1b7-43c1-b971-08b3458974bc"}
{"level":"info","ts":"2026-11-03T15:09:48.587Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"4fc69a3d-b1b7-43c1-b971-08b3458974bc","RayCluster":"counter-auth-kk8bt","State":""}
{"level":"info","ts":"2026-11-03T15:09:48.587Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"4fc69a3d-b1b7-43c1-b971-08b3458974bc","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:09:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:09:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:09:51.587Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"9e4baa1f-436a-4619-a834-ea9132f6fd82","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:09:51.587Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"9e4baa1f-436a-4619-a834-ea9132f6fd82","name":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:09:51.588Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"9e4baa1f-436a-4619-a834-ea9132f6fd82","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:09:51.588Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"9e4baa1f-436a-4619-a834-ea9132f6fd82"}
{"level":"info","ts":"2026-11-03T15:09:51.588Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"9e4baa1f-436a-4619-a834-ea9132f6fd82","RayCluster":"counter-auth-kk8bt","State":""}
{"level":"info","ts":"2026-11-03T15:09:51.588Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"9e4baa1f-436a-4619-a834-ea9132f6fd82","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:09:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:09:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:09:54.589Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"4e147ca7-5829-498b-b64b-b61758ed2b64","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:09:54.589Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"4e147ca7-5829-498b-b64b-b61758ed2b64","name":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:09:54.589Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"4e147ca7-5829-498b-b64b-b61758ed2b64","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:09:54.589Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"4e147ca7-5829-498b-b64b-b61758ed2b64"}
{"level":"info","ts":"2026-11-03T15:09:54.589Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"4e147ca7-5829-498b-b64b-b61758ed2b64","RayCluster":"counter-auth-kk8bt","State":""}
{"level":"info","ts":"2026-11-03T15:09:54.589Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"4e147ca7-5829-498b-b64b-b61758ed2b64","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:09:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:09:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:09:57.590Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"7b561e61-0fe4-41a6-9421-9aefdd2a748e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:09:57.590Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"7b561e61-0fe4-41a6-9421-9aefdd2a748e","name":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:09:57.590Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"7b561e61-0fe4-41a6-9421-9aefdd2a748e","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:09:57.590Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"7b561e61-0fe4-41a6-9421-9aefdd2a748e"}
{"level":"info","ts":"2026-11-03T15:09:57.590Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"7b561e61-0fe4-41a6-9421-9aefdd2a748e","RayCluster":"counter-auth-kk8bt","State":""}
{"level":"info","ts":"2026-11-03T15:09:57.590Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"7b561e61-0fe4-41a6-9421-9aefdd2a748e","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:09:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:09:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:10:00.592Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"9448007d-2c19-44d4-a2a5-41377f07e772","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:10:00.592Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"9448007d-2c19-44d4-a2a5-41377f07e772","name":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:10:00.592Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"9448007d-2c19-44d4-a2a5-41377f07e772","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:10:00.592Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"9448007d-2c19-44d4-a2a5-41377f07e772"}
{"level":"info","ts":"2026-11-03T15:10:00.592Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"9448007d-2c19-44d4-a2a5-41377f07e772","RayCluster":"counter-auth-kk8bt","State":""}
{"level":"info","ts":"2026-11-03T15:10:00.592Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"9448007d-2c19-44d4-a2a5-41377f07e772","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:09:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:09:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:10:03.593Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"8f3b44f0-79f3-48b7-9e54-b176f42f9d10","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:10:03.593Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"8f3b44f0-79f3-48b7-9e54-b176f42f9d10","name":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:10:03.593Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"8f3b44f0-79f3-48b7-9e54-b176f42f9d10","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:10:03.593Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"8f3b44f0-79f3-48b7-9e54-b176f42f9d10"}
{"level":"info","ts":"2026-11-03T15:10:03.593Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"8f3b44f0-79f3-48b7-9e54-b176f42f9d10","RayCluster":"counter-auth-kk8bt","State":""}
{"level":"info","ts":"2026-11-03T15:10:03.593Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"8f3b44f0-79f3-48b7-9e54-b176f42f9d10","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:09:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:09:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:10:06.594Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"b4afda45-5605-4abd-a6a5-7d1ee7b32102","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:10:06.594Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"b4afda45-5605-4abd-a6a5-7d1ee7b32102","name":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:10:06.594Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"b4afda45-5605-4abd-a6a5-7d1ee7b32102","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:10:06.594Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"b4afda45-5605-4abd-a6a5-7d1ee7b32102"}
{"level":"info","ts":"2026-11-03T15:10:06.594Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"b4afda45-5605-4abd-a6a5-7d1ee7b32102","RayCluster":"counter-auth-kk8bt","State":""}
{"level":"info","ts":"2026-11-03T15:10:06.594Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"b4afda45-5605-4abd-a6a5-7d1ee7b32102","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:09:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:09:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:10:09.595Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"a83485a6-3457-4b09-a370-c8ab61dd2851","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:10:09.595Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"a83485a6-3457-4b09-a370-c8ab61dd2851","name":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:10:09.595Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"a83485a6-3457-4b09-a370-c8ab61dd2851","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:10:09.595Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"a83485a6-3457-4b09-a370-c8ab61dd2851"}
{"level":"info","ts":"2026-11-03T15:10:09.595Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"a83485a6-3457-4b09-a370-c8ab61dd2851","RayCluster":"counter-auth-kk8bt","State":""}
{"level":"info","ts":"2026-11-03T15:10:09.595Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"a83485a6-3457-4b09-a370-c8ab61dd2851","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:09:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:09:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:10:12.596Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"afd585a9-0b99-46de-aaca-ad33389d8653","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:10:12.596Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"afd585a9-0b99-46de-aaca-ad33389d8653","name":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:10:12.596Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"afd585a9-0b99-46de-aaca-ad33389d8653","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:10:12.596Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"afd585a9-0b99-46de-aaca-ad33389d8653"}
{"level":"info","ts":"2026-11-03T15:10:12.596Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"afd585a9-0b99-46de-aaca-ad33389d8653","RayCluster":"counter-auth-kk8bt","State":""}
{"level":"info","ts":"2026-11-03T15:10:12.596Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"afd585a9-0b99-46de-aaca-ad33389d8653","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:09:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:09:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:10:15.597Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"74e632cc-02e2-4a8c-a80d-2e92661cdc18","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:10:15.597Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"74e632cc-02e2-4a8c-a80d-2e92661cdc18","name":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:10:15.597Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"74e632cc-02e2-4a8c-a80d-2e92661cdc18","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:10:15.597Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"74e632cc-02e2-4a8c-a80d-2e92661cdc18"}
{"level":"info","ts":"2026-11-03T15:10:15.597Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"74e632cc-02e2-4a8c-a80d-2e92661cdc18","RayCluster":"counter-auth-kk8bt","State":""}
{"level":"info","ts":"2026-11-03T15:10:15.597Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"74e632cc-02e2-4a8c-a80d-2e92661cdc18","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:09:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:09:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:10:18.598Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"2523a1bd-29c3-4aaf-9c28-8f6e5977b8b6","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:10:18.598Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"2523a1bd-29c3-4aaf-9c28-8f6e5977b8b6","name":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:10:18.598Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"2523a1bd-29c3-4aaf-9c28-8f6e5977b8b6","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:10:18.598Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"2523a1bd-29c3-4aaf-9c28-8f6e5977b8b6"}
{"level":"info","ts":"2026-11-03T15:10:18.598Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"2523a1bd-29c3-4aaf-9c28-8f6e5977b8b6","RayCluster":"counter-auth-kk8bt","State":""}
{"level":"info","ts":"2026-11-03T15:10:18.598Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"2523a1bd-29c3-4aaf-9c28-8f6e5977b8b6","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:09:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:09:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:10:19.755Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"070b0306-08b8-48aa-8765-7ca7fcd425b7"}
{"level":"info","ts":"2026-11-03T15:10:19.755Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"070b0306-08b8-48aa-8765-7ca7fcd425b7"}
{"level":"info","ts":"2026-11-03T15:10:19.755Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"070b0306-08b8-48aa-8765-7ca7fcd425b7","1 head service found":"counter-auth-kk8bt-head-svc"}
{"level":"info","ts":"2026-11-03T15:10:19.755Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"070b0306-08b8-48aa-8765-7ca7fcd425b7","Found 1 head Pod":"counter-auth-kk8bt-head-27gn8","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:10:19.755Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"070b0306-08b8-48aa-8765-7ca7fcd425b7","head Pod":"counter-auth-kk8bt-head-27gn8","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-kk8bt-head-27gn8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:10:19.755Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"070b0306-08b8-48aa-8765-7ca7fcd425b7","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:10:19.755Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"070b0306-08b8-48aa-8765-7ca7fcd425b7","worker Pod":"counter-auth-kk8bt-small-group-worker-k56zq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-kk8bt-small-group-worker-k56zq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:10:19.755Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"070b0306-08b8-48aa-8765-7ca7fcd425b7","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:10:19.755Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"070b0306-08b8-48aa-8765-7ca7fcd425b7","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:10:19.755Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"070b0306-08b8-48aa-8765-7ca7fcd425b7","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:10:19.755Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"070b0306-08b8-48aa-8765-7ca7fcd425b7","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:10:19.755Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"070b0306-08b8-48aa-8765-7ca7fcd425b7","podName":"counter-auth-kk8bt-small-group-worker-k56zq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:10:19.755Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"070b0306-08b8-48aa-8765-7ca7fcd425b7","podName":"counter-auth-kk8bt-small-group-worker-k56zq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:10:19.755Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"070b0306-08b8-48aa-8765-7ca7fcd425b7","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:10:19.755Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"070b0306-08b8-48aa-8765-7ca7fcd425b7","seconds":300}
{"level":"info","ts":"2026-11-03T15:10:20.715Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"a7016168-22b3-4207-851d-7aba16f7cc5b"}
{"level":"info","ts":"2026-11-03T15:10:20.715Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"a7016168-22b3-4207-851d-7aba16f7cc5b"}
{"level":"info","ts":"2026-11-03T15:10:20.715Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"a7016168-22b3-4207-851d-7aba16f7cc5b","1 head service found":"counter-auth-kk8bt-head-svc"}
{"level":"info","ts":"2026-11-03T15:10:20.715Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"a7016168-22b3-4207-851d-7aba16f7cc5b","Found 1 head Pod":"counter-auth-kk8bt-head-27gn8","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:10:20.716Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"a7016168-22b3-4207-851d-7aba16f7cc5b","head Pod":"counter-auth-kk8bt-head-27gn8","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-kk8bt-head-27gn8. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:10:20.716Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"a7016168-22b3-4207-851d-7aba16f7cc5b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:10:20.716Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"a7016168-22b3-4207-851d-7aba16f7cc5b","worker Pod":"counter-auth-kk8bt-small-group-worker-k56zq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-kk8bt-small-group-worker-k56zq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:10:20.716Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"a7016168-22b3-4207-851d-7aba16f7cc5b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:10:20.716Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"a7016168-22b3-4207-851d-7aba16f7cc5b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:10:20.716Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"a7016168-22b3-4207-851d-7aba16f7cc5b","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:10:20.716Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"a7016168-22b3-4207-851d-7aba16f7cc5b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:10:20.716Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"a7016168-22b3-4207-851d-7aba16f7cc5b","podName":"counter-auth-kk8bt-head-27gn8","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:09:13Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:10:20.716Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"a7016168-22b3-4207-851d-7aba16f7cc5b","podName":"counter-auth-kk8bt-head-27gn8","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:09:13Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:10:20.716Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"a7016168-22b3-4207-851d-7aba16f7cc5b","name":"counter-auth-kk8bt","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:09:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:10:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:10:20.777Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"aac3a4a3-078e-4a88-a143-9100b6f26686","cluster":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:10:20.778Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"3891071c-2ab0-4989-8ad9-c3bb8259d217","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:10:20.778Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"3891071c-2ab0-4989-8ad9-c3bb8259d217","name":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:10:20.778Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"3891071c-2ab0-4989-8ad9-c3bb8259d217","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:10:20.778Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"3891071c-2ab0-4989-8ad9-c3bb8259d217"}
{"level":"info","ts":"2026-11-03T15:10:20.778Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"3891071c-2ab0-4989-8ad9-c3bb8259d217","RayCluster":"counter-auth-kk8bt","State":""}
{"level":"info","ts":"2026-11-03T15:10:20.778Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"3891071c-2ab0-4989-8ad9-c3bb8259d217","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:09:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:10:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:10:20.778Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"3891071c-2ab0-4989-8ad9-c3bb8259d217","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:10:20.838Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"67412756-ac70-49bd-8c94-49f8a56dec08","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:10:20.838Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"67412756-ac70-49bd-8c94-49f8a56dec08","name":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:10:20.838Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"67412756-ac70-49bd-8c94-49f8a56dec08","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:10:20.839Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"67412756-ac70-49bd-8c94-49f8a56dec08"}
{"level":"info","ts":"2026-11-03T15:10:20.839Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"67412756-ac70-49bd-8c94-49f8a56dec08","RayCluster":"counter-auth-kk8bt","State":""}
{"level":"info","ts":"2026-11-03T15:10:20.839Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"67412756-ac70-49bd-8c94-49f8a56dec08","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:10:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:10:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:10:22.777Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"408f4ec5-de98-4ac3-aa95-89644fd94cd9"}
{"level":"info","ts":"2026-11-03T15:10:22.777Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"408f4ec5-de98-4ac3-aa95-89644fd94cd9"}
{"level":"info","ts":"2026-11-03T15:10:22.777Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"408f4ec5-de98-4ac3-aa95-89644fd94cd9","1 head service found":"counter-auth-kk8bt-head-svc"}
{"level":"info","ts":"2026-11-03T15:10:22.777Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"408f4ec5-de98-4ac3-aa95-89644fd94cd9","Found 1 head Pod":"counter-auth-kk8bt-head-27gn8","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:10:22.777Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"408f4ec5-de98-4ac3-aa95-89644fd94cd9","head Pod":"counter-auth-kk8bt-head-27gn8","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-kk8bt-head-27gn8. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:10:22.777Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"408f4ec5-de98-4ac3-aa95-89644fd94cd9","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:10:22.777Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"408f4ec5-de98-4ac3-aa95-89644fd94cd9","worker Pod":"counter-auth-kk8bt-small-group-worker-k56zq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-kk8bt-small-group-worker-k56zq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:10:22.777Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"408f4ec5-de98-4ac3-aa95-89644fd94cd9","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:10:22.777Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"408f4ec5-de98-4ac3-aa95-89644fd94cd9","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:10:22.777Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"408f4ec5-de98-4ac3-aa95-89644fd94cd9","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:10:22.777Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"408f4ec5-de98-4ac3-aa95-89644fd94cd9","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:10:22.777Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"408f4ec5-de98-4ac3-aa95-89644fd94cd9","podName":"counter-auth-kk8bt-head-27gn8","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:09:13Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:10:22.777Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"408f4ec5-de98-4ac3-aa95-89644fd94cd9","podName":"counter-auth-kk8bt-head-27gn8","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:09:13Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:10:22.777Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"408f4ec5-de98-4ac3-aa95-89644fd94cd9","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:10:22.777Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"408f4ec5-de98-4ac3-aa95-89644fd94cd9","seconds":300}
{"level":"info","ts":"2026-11-03T15:10:23.839Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"c3e98098-95b5-4cce-8f67-b42629c38aef","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:10:23.839Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"c3e98098-95b5-4cce-8f67-b42629c38aef","name":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:10:23.840Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"c3e98098-95b5-4cce-8f67-b42629c38aef","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:10:23.840Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"c3e98098-95b5-4cce-8f67-b42629c38aef"}
{"level":"info","ts":"2026-11-03T15:10:23.840Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"c3e98098-95b5-4cce-8f67-b42629c38aef","RayCluster":"counter-auth-kk8bt","State":""}
{"level":"info","ts":"2026-11-03T15:10:23.840Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"c3e98098-95b5-4cce-8f67-b42629c38aef","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:10:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:10:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:10:26.840Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"82686653-37e6-4e1d-bd1d-fc1be13f4669","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:10:26.840Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"82686653-37e6-4e1d-bd1d-fc1be13f4669","name":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:10:26.841Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"82686653-37e6-4e1d-bd1d-fc1be13f4669","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:10:26.841Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"82686653-37e6-4e1d-bd1d-fc1be13f4669"}
{"level":"info","ts":"2026-11-03T15:10:26.841Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"82686653-37e6-4e1d-bd1d-fc1be13f4669","RayCluster":"counter-auth-kk8bt","State":""}
{"level":"info","ts":"2026-11-03T15:10:26.841Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"82686653-37e6-4e1d-bd1d-fc1be13f4669","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:10:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:10:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:10:29.842Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"8a603cd5-a3e1-4591-85ba-418f7323c72d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:10:29.842Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"8a603cd5-a3e1-4591-85ba-418f7323c72d","name":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:10:29.842Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"8a603cd5-a3e1-4591-85ba-418f7323c72d","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:10:29.842Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"8a603cd5-a3e1-4591-85ba-418f7323c72d"}
{"level":"info","ts":"2026-11-03T15:10:29.842Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"8a603cd5-a3e1-4591-85ba-418f7323c72d","RayCluster":"counter-auth-kk8bt","State":""}
{"level":"info","ts":"2026-11-03T15:10:29.842Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"8a603cd5-a3e1-4591-85ba-418f7323c72d","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:10:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:10:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:10:31.772Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"2f3a77aa-3be5-48bc-8354-b9080770c20c"}
{"level":"info","ts":"2026-11-03T15:10:31.772Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"2f3a77aa-3be5-48bc-8354-b9080770c20c"}
{"level":"info","ts":"2026-11-03T15:10:31.772Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"2f3a77aa-3be5-48bc-8354-b9080770c20c","1 head service found":"counter-auth-kk8bt-head-svc"}
{"level":"info","ts":"2026-11-03T15:10:31.772Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"2f3a77aa-3be5-48bc-8354-b9080770c20c","Found 1 head Pod":"counter-auth-kk8bt-head-27gn8","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:10:31.772Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"2f3a77aa-3be5-48bc-8354-b9080770c20c","head Pod":"counter-auth-kk8bt-head-27gn8","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-kk8bt-head-27gn8. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:10:31.772Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"2f3a77aa-3be5-48bc-8354-b9080770c20c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:10:31.772Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"2f3a77aa-3be5-48bc-8354-b9080770c20c","worker Pod":"counter-auth-kk8bt-small-group-worker-k56zq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-kk8bt-small-group-worker-k56zq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:10:31.772Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"2f3a77aa-3be5-48bc-8354-b9080770c20c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:10:31.772Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"2f3a77aa-3be5-48bc-8354-b9080770c20c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:10:31.772Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"2f3a77aa-3be5-48bc-8354-b9080770c20c","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:10:31.772Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"2f3a77aa-3be5-48bc-8354-b9080770c20c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:10:31.772Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"2f3a77aa-3be5-48bc-8354-b9080770c20c","podName":"counter-auth-kk8bt-small-group-worker-k56zq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:10:31.772Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"2f3a77aa-3be5-48bc-8354-b9080770c20c","podName":"counter-auth-kk8bt-small-group-worker-k56zq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:10:31.773Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"2f3a77aa-3be5-48bc-8354-b9080770c20c","name":"counter-auth-kk8bt","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:10:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:10:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:10:31Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:10:31.834Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"aebfe711-3dd9-47b4-9e06-3144b29421ce","cluster":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:10:31.834Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"109436aa-c2aa-418d-9396-1689fce3b920","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:10:31.834Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"109436aa-c2aa-418d-9396-1689fce3b920","name":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:10:31.834Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"109436aa-c2aa-418d-9396-1689fce3b920","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:10:31.834Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"109436aa-c2aa-418d-9396-1689fce3b920"}
{"level":"info","ts":"2026-11-03T15:10:31.834Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"109436aa-c2aa-418d-9396-1689fce3b920","RayCluster":"counter-auth-kk8bt","State":""}
{"level":"info","ts":"2026-11-03T15:10:31.834Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"109436aa-c2aa-418d-9396-1689fce3b920","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:10:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:10:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:10:31Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:10:31.834Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"109436aa-c2aa-418d-9396-1689fce3b920","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:10:31.895Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"4d97c3cf-486b-4f46-9008-a31e8639bf26","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:10:31.895Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"4d97c3cf-486b-4f46-9008-a31e8639bf26","name":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:10:31.895Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"4d97c3cf-486b-4f46-9008-a31e8639bf26","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:10:31.895Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"4d97c3cf-486b-4f46-9008-a31e8639bf26"}
{"level":"info","ts":"2026-11-03T15:10:31.895Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"4d97c3cf-486b-4f46-9008-a31e8639bf26","RayCluster":"counter-auth-kk8bt","State":""}
{"level":"info","ts":"2026-11-03T15:10:31.895Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"4d97c3cf-486b-4f46-9008-a31e8639bf26","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:10:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:10:31Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:10:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:10:31Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:10:33.834Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"b9c25d1f-0c40-43e9-b099-a16af3420659"}
{"level":"info","ts":"2026-11-03T15:10:33.834Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"b9c25d1f-0c40-43e9-b099-a16af3420659"}
{"level":"info","ts":"2026-11-03T15:10:33.834Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"b9c25d1f-0c40-43e9-b099-a16af3420659","1 head service found":"counter-auth-kk8bt-head-svc"}
{"level":"info","ts":"2026-11-03T15:10:33.834Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"b9c25d1f-0c40-43e9-b099-a16af3420659","Found 1 head Pod":"counter-auth-kk8bt-head-27gn8","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:10:33.834Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"b9c25d1f-0c40-43e9-b099-a16af3420659","head Pod":"counter-auth-kk8bt-head-27gn8","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-kk8bt-head-27gn8. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:10:33.834Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"b9c25d1f-0c40-43e9-b099-a16af3420659","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:10:33.834Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"b9c25d1f-0c40-43e9-b099-a16af3420659","worker Pod":"counter-auth-kk8bt-small-group-worker-k56zq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-kk8bt-small-group-worker-k56zq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:10:33.834Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"b9c25d1f-0c40-43e9-b099-a16af3420659","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:10:33.834Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"b9c25d1f-0c40-43e9-b099-a16af3420659","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:10:33.834Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"b9c25d1f-0c40-43e9-b099-a16af3420659","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:10:33.834Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"b9c25d1f-0c40-43e9-b099-a16af3420659","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:10:33.834Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"b9c25d1f-0c40-43e9-b099-a16af3420659","podName":"counter-auth-kk8bt-small-group-worker-k56zq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:10:33.834Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"b9c25d1f-0c40-43e9-b099-a16af3420659","podName":"counter-auth-kk8bt-small-group-worker-k56zq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:10:33.834Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"b9c25d1f-0c40-43e9-b099-a16af3420659","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:10:33.834Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"b9c25d1f-0c40-43e9-b099-a16af3420659","seconds":300}
{"level":"info","ts":"2026-11-03T15:10:34.896Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"9afef55e-d8a6-43a8-911e-066ce3df9b2a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:10:34.896Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"9afef55e-d8a6-43a8-911e-066ce3df9b2a","name":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:10:34.896Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"9afef55e-d8a6-43a8-911e-066ce3df9b2a","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:10:34.896Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"9afef55e-d8a6-43a8-911e-066ce3df9b2a"}
{"level":"info","ts":"2026-11-03T15:10:34.896Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"9afef55e-d8a6-43a8-911e-066ce3df9b2a","RayCluster":"counter-auth-kk8bt","State":""}
{"level":"info","ts":"2026-11-03T15:10:34.896Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"9afef55e-d8a6-43a8-911e-066ce3df9b2a","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:10:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:10:31Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:10:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:10:31Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:10:37.896Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"f74a8b52-20a6-4359-86cb-e95dee3c8594","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:10:37.896Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"f74a8b52-20a6-4359-86cb-e95dee3c8594","name":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:10:37.897Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"f74a8b52-20a6-4359-86cb-e95dee3c8594","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:10:37.897Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"f74a8b52-20a6-4359-86cb-e95dee3c8594"}
{"level":"info","ts":"2026-11-03T15:10:37.897Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"f74a8b52-20a6-4359-86cb-e95dee3c8594","RayCluster":"counter-auth-kk8bt","State":""}
{"level":"info","ts":"2026-11-03T15:10:37.897Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"f74a8b52-20a6-4359-86cb-e95dee3c8594","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:10:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:10:31Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:10:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:10:31Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:10:40.897Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"beef888e-fba1-4da6-af87-e0d811ba1f2b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:10:40.897Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"beef888e-fba1-4da6-af87-e0d811ba1f2b","name":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:10:40.897Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"beef888e-fba1-4da6-af87-e0d811ba1f2b","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:10:40.898Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"beef888e-fba1-4da6-af87-e0d811ba1f2b"}
{"level":"info","ts":"2026-11-03T15:10:40.898Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"beef888e-fba1-4da6-af87-e0d811ba1f2b","RayCluster":"counter-auth-kk8bt","State":""}
{"level":"info","ts":"2026-11-03T15:10:40.898Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"beef888e-fba1-4da6-af87-e0d811ba1f2b","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:10:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:10:31Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:10:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:10:31Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"error","ts":"2026-11-03T15:10:41.509Z","logger":"events","msg":"Server rejected event (will not retry!)","event":{"namespace":"test-ns-g9m7p","name":"raycluster.18e490f229b8c245"},"error":"namespaces \"test-ns-g9m7p\" not found","stacktrace":"k8s.io/client-go/tools/events.recordEvent\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:270\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).finishSeries\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:153\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext.func1\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:255\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:256\nk8s.io/apimachinery/pkg/util/wait.JitterUntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:223\nk8s.io/apimachinery/pkg/util/wait.UntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:172"}
{"level":"info","ts":"2026-11-03T15:10:41.606Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"369c9721-a1f2-4cf9-9575-e82468a14ec1"}
{"level":"info","ts":"2026-11-03T15:10:41.606Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"369c9721-a1f2-4cf9-9575-e82468a14ec1"}
{"level":"info","ts":"2026-11-03T15:10:41.606Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"369c9721-a1f2-4cf9-9575-e82468a14ec1","1 head service found":"counter-auth-kk8bt-head-svc"}
{"level":"info","ts":"2026-11-03T15:10:41.606Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"369c9721-a1f2-4cf9-9575-e82468a14ec1","Found 1 head Pod":"counter-auth-kk8bt-head-27gn8","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:10:41.606Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"369c9721-a1f2-4cf9-9575-e82468a14ec1","head Pod":"counter-auth-kk8bt-head-27gn8","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-kk8bt-head-27gn8. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:10:41.606Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"369c9721-a1f2-4cf9-9575-e82468a14ec1","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:10:41.606Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"369c9721-a1f2-4cf9-9575-e82468a14ec1","worker Pod":"counter-auth-kk8bt-small-group-worker-k56zq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-kk8bt-small-group-worker-k56zq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:10:41.606Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"369c9721-a1f2-4cf9-9575-e82468a14ec1","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:10:41.606Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"369c9721-a1f2-4cf9-9575-e82468a14ec1","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:10:41.606Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"369c9721-a1f2-4cf9-9575-e82468a14ec1","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:10:41.606Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"369c9721-a1f2-4cf9-9575-e82468a14ec1","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:10:41.606Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"369c9721-a1f2-4cf9-9575-e82468a14ec1","podName":"counter-auth-kk8bt-small-group-worker-k56zq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:10:41.606Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"369c9721-a1f2-4cf9-9575-e82468a14ec1","podName":"counter-auth-kk8bt-small-group-worker-k56zq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:10:41.607Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"369c9721-a1f2-4cf9-9575-e82468a14ec1","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:10:41.607Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"369c9721-a1f2-4cf9-9575-e82468a14ec1","seconds":300}
{"level":"error","ts":"2026-11-03T15:10:41.660Z","logger":"events","msg":"Server rejected event (will not retry!)","event":{"namespace":"test-ns-g9m7p","name":"raycluster.18e490f333d556e8"},"error":"namespaces \"test-ns-g9m7p\" not found","stacktrace":"k8s.io/client-go/tools/events.recordEvent\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:270\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).finishSeries\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:153\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext.func1\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:255\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:256\nk8s.io/apimachinery/pkg/util/wait.JitterUntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:223\nk8s.io/apimachinery/pkg/util/wait.UntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:172"}
{"level":"error","ts":"2026-11-03T15:10:41.811Z","logger":"events","msg":"Server rejected event (will not retry!)","event":{"namespace":"test-ns-g9m7p","name":"raycluster.18e491317910fd0f"},"error":"namespaces \"test-ns-g9m7p\" not found","stacktrace":"k8s.io/client-go/tools/events.recordEvent\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:270\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).finishSeries\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:153\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext.func1\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:255\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:256\nk8s.io/apimachinery/pkg/util/wait.JitterUntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:223\nk8s.io/apimachinery/pkg/util/wait.UntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:172"}
{"level":"error","ts":"2026-11-03T15:10:41.963Z","logger":"events","msg":"Server rejected event (will not retry!)","event":{"namespace":"test-ns-g9m7p","name":"raycluster.18e49132590e1510"},"error":"namespaces \"test-ns-g9m7p\" not found","stacktrace":"k8s.io/client-go/tools/events.recordEvent\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:270\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).finishSeries\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:153\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext.func1\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:255\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:256\nk8s.io/apimachinery/pkg/util/wait.JitterUntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:223\nk8s.io/apimachinery/pkg/util/wait.UntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:172"}
{"level":"error","ts":"2026-11-03T15:10:42.113Z","logger":"events","msg":"Server rejected event (will not retry!)","event":{"namespace":"test-ns-g9m7p","name":"raycluster.18e49110c0dfb364"},"error":"namespaces \"test-ns-g9m7p\" not found","stacktrace":"k8s.io/client-go/tools/events.recordEvent\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:270\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).finishSeries\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:153\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext.func1\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:255\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:256\nk8s.io/apimachinery/pkg/util/wait.JitterUntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:223\nk8s.io/apimachinery/pkg/util/wait.UntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:172"}
{"level":"info","ts":"2026-11-03T15:10:42.609Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"7563e893-fcdf-48e2-9cb7-dfac7f832ec8"}
{"level":"info","ts":"2026-11-03T15:10:42.609Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"7563e893-fcdf-48e2-9cb7-dfac7f832ec8"}
{"level":"info","ts":"2026-11-03T15:10:42.609Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"7563e893-fcdf-48e2-9cb7-dfac7f832ec8","1 head service found":"counter-auth-kk8bt-head-svc"}
{"level":"info","ts":"2026-11-03T15:10:42.609Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"7563e893-fcdf-48e2-9cb7-dfac7f832ec8","Found 1 head Pod":"counter-auth-kk8bt-head-27gn8","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:10:42.609Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"7563e893-fcdf-48e2-9cb7-dfac7f832ec8","head Pod":"counter-auth-kk8bt-head-27gn8","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-kk8bt-head-27gn8. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:10:42.609Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"7563e893-fcdf-48e2-9cb7-dfac7f832ec8","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:10:42.609Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"7563e893-fcdf-48e2-9cb7-dfac7f832ec8","worker Pod":"counter-auth-kk8bt-small-group-worker-k56zq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-kk8bt-small-group-worker-k56zq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:10:42.609Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"7563e893-fcdf-48e2-9cb7-dfac7f832ec8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:10:42.609Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"7563e893-fcdf-48e2-9cb7-dfac7f832ec8","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:10:42.609Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"7563e893-fcdf-48e2-9cb7-dfac7f832ec8","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:10:42.609Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"7563e893-fcdf-48e2-9cb7-dfac7f832ec8","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:10:42.609Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"7563e893-fcdf-48e2-9cb7-dfac7f832ec8","podName":"counter-auth-kk8bt-small-group-worker-k56zq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:10:42.609Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"7563e893-fcdf-48e2-9cb7-dfac7f832ec8","podName":"counter-auth-kk8bt-small-group-worker-k56zq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:10:42.609Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"7563e893-fcdf-48e2-9cb7-dfac7f832ec8","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:10:42.609Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"7563e893-fcdf-48e2-9cb7-dfac7f832ec8","seconds":300}
{"level":"info","ts":"2026-11-03T15:10:43.898Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"a7e859b0-5a53-4430-b0dc-e758ad3916d6","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:10:43.898Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"a7e859b0-5a53-4430-b0dc-e758ad3916d6","name":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:10:43.898Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"a7e859b0-5a53-4430-b0dc-e758ad3916d6","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:10:43.898Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"a7e859b0-5a53-4430-b0dc-e758ad3916d6"}
{"level":"info","ts":"2026-11-03T15:10:43.898Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"a7e859b0-5a53-4430-b0dc-e758ad3916d6","RayCluster":"counter-auth-kk8bt","State":""}
{"level":"info","ts":"2026-11-03T15:10:43.898Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"a7e859b0-5a53-4430-b0dc-e758ad3916d6","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:10:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:10:31Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:10:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:10:31Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:10:46.899Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"4f2a1620-7177-469b-aff8-ea8dfbe2da15","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:10:46.899Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"4f2a1620-7177-469b-aff8-ea8dfbe2da15","name":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:10:46.899Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"4f2a1620-7177-469b-aff8-ea8dfbe2da15","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:10:46.899Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"4f2a1620-7177-469b-aff8-ea8dfbe2da15"}
{"level":"info","ts":"2026-11-03T15:10:46.899Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"4f2a1620-7177-469b-aff8-ea8dfbe2da15","RayCluster":"counter-auth-kk8bt","State":""}
{"level":"info","ts":"2026-11-03T15:10:46.899Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"4f2a1620-7177-469b-aff8-ea8dfbe2da15","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:10:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:10:31Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:10:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:10:31Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:10:49.900Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"1e1d94ba-914b-4242-812a-b3ee3520a4e9","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:10:49.900Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"1e1d94ba-914b-4242-812a-b3ee3520a4e9","name":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:10:49.900Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"1e1d94ba-914b-4242-812a-b3ee3520a4e9","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:10:49.901Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"1e1d94ba-914b-4242-812a-b3ee3520a4e9"}
{"level":"info","ts":"2026-11-03T15:10:49.901Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"1e1d94ba-914b-4242-812a-b3ee3520a4e9","RayCluster":"counter-auth-kk8bt","State":""}
{"level":"info","ts":"2026-11-03T15:10:49.901Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"1e1d94ba-914b-4242-812a-b3ee3520a4e9","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:10:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:10:31Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:10:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:10:31Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:10:51.697Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"e8cab7e1-bd19-4078-b9f6-d3515a65734c"}
{"level":"info","ts":"2026-11-03T15:10:51.697Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"e8cab7e1-bd19-4078-b9f6-d3515a65734c"}
{"level":"info","ts":"2026-11-03T15:10:51.697Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"e8cab7e1-bd19-4078-b9f6-d3515a65734c","1 head service found":"counter-auth-kk8bt-head-svc"}
{"level":"info","ts":"2026-11-03T15:10:51.697Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"e8cab7e1-bd19-4078-b9f6-d3515a65734c","Found 1 head Pod":"counter-auth-kk8bt-head-27gn8","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:10:51.697Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"e8cab7e1-bd19-4078-b9f6-d3515a65734c","head Pod":"counter-auth-kk8bt-head-27gn8","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-kk8bt-head-27gn8. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:10:51.697Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"e8cab7e1-bd19-4078-b9f6-d3515a65734c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:10:51.697Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"e8cab7e1-bd19-4078-b9f6-d3515a65734c","worker Pod":"counter-auth-kk8bt-small-group-worker-k56zq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-kk8bt-small-group-worker-k56zq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:10:51.697Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"e8cab7e1-bd19-4078-b9f6-d3515a65734c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:10:51.697Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"e8cab7e1-bd19-4078-b9f6-d3515a65734c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:10:51.697Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"e8cab7e1-bd19-4078-b9f6-d3515a65734c","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:10:51.697Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"e8cab7e1-bd19-4078-b9f6-d3515a65734c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:10:51.697Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"e8cab7e1-bd19-4078-b9f6-d3515a65734c","podName":"counter-auth-kk8bt-small-group-worker-k56zq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:10:51.697Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"e8cab7e1-bd19-4078-b9f6-d3515a65734c","podName":"counter-auth-kk8bt-small-group-worker-k56zq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:10:51.697Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"e8cab7e1-bd19-4078-b9f6-d3515a65734c","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:10:51.697Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"e8cab7e1-bd19-4078-b9f6-d3515a65734c","seconds":300}
{"level":"info","ts":"2026-11-03T15:10:52.703Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"af0f2d84-940b-4db9-843e-29030b518aa4"}
{"level":"info","ts":"2026-11-03T15:10:52.703Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"af0f2d84-940b-4db9-843e-29030b518aa4"}
{"level":"info","ts":"2026-11-03T15:10:52.703Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"af0f2d84-940b-4db9-843e-29030b518aa4","1 head service found":"counter-auth-kk8bt-head-svc"}
{"level":"info","ts":"2026-11-03T15:10:52.703Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"af0f2d84-940b-4db9-843e-29030b518aa4","Found 1 head Pod":"counter-auth-kk8bt-head-27gn8","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:10:52.703Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"af0f2d84-940b-4db9-843e-29030b518aa4","head Pod":"counter-auth-kk8bt-head-27gn8","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-kk8bt-head-27gn8. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:10:52.703Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"af0f2d84-940b-4db9-843e-29030b518aa4","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:10:52.703Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"af0f2d84-940b-4db9-843e-29030b518aa4","worker Pod":"counter-auth-kk8bt-small-group-worker-k56zq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-kk8bt-small-group-worker-k56zq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:10:52.703Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"af0f2d84-940b-4db9-843e-29030b518aa4","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:10:52.703Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"af0f2d84-940b-4db9-843e-29030b518aa4","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:10:52.703Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"af0f2d84-940b-4db9-843e-29030b518aa4","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:10:52.703Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"af0f2d84-940b-4db9-843e-29030b518aa4","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:10:52.703Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"af0f2d84-940b-4db9-843e-29030b518aa4","podName":"counter-auth-kk8bt-small-group-worker-k56zq","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:09:13Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:10:52.703Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"af0f2d84-940b-4db9-843e-29030b518aa4","podName":"counter-auth-kk8bt-small-group-worker-k56zq","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:09:13Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:10:52.703Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"af0f2d84-940b-4db9-843e-29030b518aa4","name":"counter-auth-kk8bt","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:10:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:10:31Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:10:52Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:10:31Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:10:52.765Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"66282972-2718-4128-8c70-774374756f01","cluster":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:10:52.765Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"25de7655-f259-4b3f-9b2d-a70e380b982f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:10:52.765Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"25de7655-f259-4b3f-9b2d-a70e380b982f","name":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:10:52.765Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"25de7655-f259-4b3f-9b2d-a70e380b982f","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:10:52.766Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"25de7655-f259-4b3f-9b2d-a70e380b982f"}
{"level":"info","ts":"2026-11-03T15:10:52.766Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"25de7655-f259-4b3f-9b2d-a70e380b982f","RayCluster":"counter-auth-kk8bt","State":""}
{"level":"info","ts":"2026-11-03T15:10:52.766Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"25de7655-f259-4b3f-9b2d-a70e380b982f","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:10:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:10:31Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:10:52Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:10:31Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:10:52.766Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"25de7655-f259-4b3f-9b2d-a70e380b982f","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:10:52.826Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"68557c7e-5bfb-4ecd-9a44-8703ceb311bc","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:10:52.826Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"68557c7e-5bfb-4ecd-9a44-8703ceb311bc","name":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:10:52.826Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"68557c7e-5bfb-4ecd-9a44-8703ceb311bc","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:10:52.827Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"68557c7e-5bfb-4ecd-9a44-8703ceb311bc"}
{"level":"info","ts":"2026-11-03T15:10:52.827Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"68557c7e-5bfb-4ecd-9a44-8703ceb311bc","RayCluster":"counter-auth-kk8bt","State":""}
{"level":"info","ts":"2026-11-03T15:10:52.827Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"68557c7e-5bfb-4ecd-9a44-8703ceb311bc","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:10:52Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:10:31Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:10:52Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:10:31Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:10:54.765Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"2ddd107b-43ef-44c7-9960-9bf4f1e9665a"}
{"level":"info","ts":"2026-11-03T15:10:54.765Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"2ddd107b-43ef-44c7-9960-9bf4f1e9665a"}
{"level":"info","ts":"2026-11-03T15:10:54.765Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"2ddd107b-43ef-44c7-9960-9bf4f1e9665a","1 head service found":"counter-auth-kk8bt-head-svc"}
{"level":"info","ts":"2026-11-03T15:10:54.765Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"2ddd107b-43ef-44c7-9960-9bf4f1e9665a","Found 1 head Pod":"counter-auth-kk8bt-head-27gn8","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:10:54.765Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"2ddd107b-43ef-44c7-9960-9bf4f1e9665a","head Pod":"counter-auth-kk8bt-head-27gn8","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-kk8bt-head-27gn8. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:10:54.765Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"2ddd107b-43ef-44c7-9960-9bf4f1e9665a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:10:54.765Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"2ddd107b-43ef-44c7-9960-9bf4f1e9665a","worker Pod":"counter-auth-kk8bt-small-group-worker-k56zq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-kk8bt-small-group-worker-k56zq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:10:54.765Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"2ddd107b-43ef-44c7-9960-9bf4f1e9665a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:10:54.765Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"2ddd107b-43ef-44c7-9960-9bf4f1e9665a","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:10:54.765Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"2ddd107b-43ef-44c7-9960-9bf4f1e9665a","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:10:54.765Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"2ddd107b-43ef-44c7-9960-9bf4f1e9665a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:10:54.766Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"2ddd107b-43ef-44c7-9960-9bf4f1e9665a","podName":"counter-auth-kk8bt-small-group-worker-k56zq","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:09:13Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:10:54.766Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"2ddd107b-43ef-44c7-9960-9bf4f1e9665a","podName":"counter-auth-kk8bt-small-group-worker-k56zq","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:09:13Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:10:54.766Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"2ddd107b-43ef-44c7-9960-9bf4f1e9665a","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:10:54.766Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"2ddd107b-43ef-44c7-9960-9bf4f1e9665a","seconds":300}
{"level":"info","ts":"2026-11-03T15:10:55.827Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"dda15026-3c6a-4677-bbed-d2e137e437cd","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:10:55.827Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"dda15026-3c6a-4677-bbed-d2e137e437cd","name":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:10:55.827Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"dda15026-3c6a-4677-bbed-d2e137e437cd","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:10:55.827Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"dda15026-3c6a-4677-bbed-d2e137e437cd"}
{"level":"info","ts":"2026-11-03T15:10:55.827Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"dda15026-3c6a-4677-bbed-d2e137e437cd","RayCluster":"counter-auth-kk8bt","State":""}
{"level":"info","ts":"2026-11-03T15:10:55.827Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"dda15026-3c6a-4677-bbed-d2e137e437cd","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:10:52Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:10:31Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:10:52Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:10:31Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:10:58.828Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"962e3f52-1c98-4c03-93f9-a3844c165c1a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:10:58.828Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"962e3f52-1c98-4c03-93f9-a3844c165c1a","name":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:10:58.829Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"962e3f52-1c98-4c03-93f9-a3844c165c1a","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:10:58.829Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"962e3f52-1c98-4c03-93f9-a3844c165c1a"}
{"level":"info","ts":"2026-11-03T15:10:58.829Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"962e3f52-1c98-4c03-93f9-a3844c165c1a","RayCluster":"counter-auth-kk8bt","State":""}
{"level":"info","ts":"2026-11-03T15:10:58.829Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"962e3f52-1c98-4c03-93f9-a3844c165c1a","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:10:52Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:10:31Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:10:52Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:10:31Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:11:01.829Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"b8b94c59-169d-4f04-b5e6-65059f85c550","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:11:01.829Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"b8b94c59-169d-4f04-b5e6-65059f85c550","name":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:11:01.830Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"b8b94c59-169d-4f04-b5e6-65059f85c550","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:11:01.830Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"b8b94c59-169d-4f04-b5e6-65059f85c550"}
{"level":"info","ts":"2026-11-03T15:11:01.830Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"b8b94c59-169d-4f04-b5e6-65059f85c550","RayCluster":"counter-auth-kk8bt","State":""}
{"level":"info","ts":"2026-11-03T15:11:01.830Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"b8b94c59-169d-4f04-b5e6-65059f85c550","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:10:52Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:10:31Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:10:52Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:10:31Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:11:03.712Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"81708ea2-fc1e-4f85-8fd1-a0aaa3337e05"}
{"level":"info","ts":"2026-11-03T15:11:03.712Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"81708ea2-fc1e-4f85-8fd1-a0aaa3337e05"}
{"level":"info","ts":"2026-11-03T15:11:03.712Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"81708ea2-fc1e-4f85-8fd1-a0aaa3337e05","1 head service found":"counter-auth-kk8bt-head-svc"}
{"level":"info","ts":"2026-11-03T15:11:03.712Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"81708ea2-fc1e-4f85-8fd1-a0aaa3337e05","Found 1 head Pod":"counter-auth-kk8bt-head-27gn8","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:11:03.713Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"81708ea2-fc1e-4f85-8fd1-a0aaa3337e05","head Pod":"counter-auth-kk8bt-head-27gn8","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-kk8bt-head-27gn8. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:11:03.713Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"81708ea2-fc1e-4f85-8fd1-a0aaa3337e05","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:11:03.713Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"81708ea2-fc1e-4f85-8fd1-a0aaa3337e05","worker Pod":"counter-auth-kk8bt-small-group-worker-k56zq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-kk8bt-small-group-worker-k56zq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:11:03.713Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"81708ea2-fc1e-4f85-8fd1-a0aaa3337e05","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:11:03.713Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"81708ea2-fc1e-4f85-8fd1-a0aaa3337e05","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:11:03.713Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"81708ea2-fc1e-4f85-8fd1-a0aaa3337e05","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:11:03.713Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"81708ea2-fc1e-4f85-8fd1-a0aaa3337e05","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:11:03.713Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"81708ea2-fc1e-4f85-8fd1-a0aaa3337e05","name":"counter-auth-kk8bt","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:10:52Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:10:31Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:03Z","stateTransitionTimes":{"ready":"2026-11-03T15:11:03Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:10:31Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:11:03Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:11:03.774Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"04e56552-6266-4be7-b0ed-87062637e2dc","cluster":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:11:03.774Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"4406f2e5-c2c0-4fbb-a262-8a0f3e9a94bc","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:11:03.774Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"4406f2e5-c2c0-4fbb-a262-8a0f3e9a94bc","name":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:11:03.774Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"4406f2e5-c2c0-4fbb-a262-8a0f3e9a94bc","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:11:03.774Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"4406f2e5-c2c0-4fbb-a262-8a0f3e9a94bc"}
{"level":"info","ts":"2026-11-03T15:11:03.774Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"4406f2e5-c2c0-4fbb-a262-8a0f3e9a94bc","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:10:52Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:10:31Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","dashboardURL":"counter-auth-kk8bt-head-svc.test-ns-mvz6h.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:10:52Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:10:31Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:11:03.774Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"4406f2e5-c2c0-4fbb-a262-8a0f3e9a94bc","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-11-03T15:11:03.835Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"a97b4d53-56af-4af9-af23-f580a4ce01b1","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:11:03.835Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"a97b4d53-56af-4af9-af23-f580a4ce01b1","name":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:11:03.835Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"a97b4d53-56af-4af9-af23-f580a4ce01b1","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:11:03.835Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"a97b4d53-56af-4af9-af23-f580a4ce01b1"}
{"level":"info","ts":"2026-11-03T15:11:03.835Z","logger":"controllers.RayJob","msg":"Create a Kubernetes Service","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"a97b4d53-56af-4af9-af23-f580a4ce01b1","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:11:03.907Z","logger":"controllers.RayJob","msg":"The Ray job was not found. Submit a Ray job via an HTTP request.","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"a97b4d53-56af-4af9-af23-f580a4ce01b1","JobId":"counter-auth-lw9mj"}
{"level":"info","ts":"2026-11-03T15:11:04.731Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"df2bf14b-8899-43e8-a2e8-ea54dce5012c","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:11:04.731Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"df2bf14b-8899-43e8-a2e8-ea54dce5012c","name":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:11:04.731Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"df2bf14b-8899-43e8-a2e8-ea54dce5012c","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:11:04.731Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"df2bf14b-8899-43e8-a2e8-ea54dce5012c"}
{"level":"info","ts":"2026-11-03T15:11:04.731Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"df2bf14b-8899-43e8-a2e8-ea54dce5012c","rayCluster":"counter-auth-kk8bt","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:11:04.734Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"df2bf14b-8899-43e8-a2e8-ea54dce5012c","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","dashboardURL":"counter-auth-kk8bt-head-svc.test-ns-mvz6h.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:10:52Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:10:31Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:11:04Z"},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","dashboardURL":"counter-auth-kk8bt-head-svc.test-ns-mvz6h.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:03Z","stateTransitionTimes":{"ready":"2026-11-03T15:11:03Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:10:31Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:11:03Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:11:04.734Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"df2bf14b-8899-43e8-a2e8-ea54dce5012c","old JobStatus":"","new JobStatus":"PENDING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-11-03T15:11:04.795Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"11dc17e6-4582-4fc4-a398-5bea319beb6b","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:11:04.795Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"11dc17e6-4582-4fc4-a398-5bea319beb6b","name":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:11:04.796Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"11dc17e6-4582-4fc4-a398-5bea319beb6b","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:11:04.796Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"11dc17e6-4582-4fc4-a398-5bea319beb6b"}
{"level":"info","ts":"2026-11-03T15:11:04.796Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"11dc17e6-4582-4fc4-a398-5bea319beb6b","rayCluster":"counter-auth-kk8bt","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:11:04.799Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"11dc17e6-4582-4fc4-a398-5bea319beb6b","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:11:04Z"},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","dashboardURL":"counter-auth-kk8bt-head-svc.test-ns-mvz6h.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:03Z","stateTransitionTimes":{"ready":"2026-11-03T15:11:03Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:10:31Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:11:03Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:11:04Z"},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","dashboardURL":"counter-auth-kk8bt-head-svc.test-ns-mvz6h.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:03Z","stateTransitionTimes":{"ready":"2026-11-03T15:11:03Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:10:31Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:11:03Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:11:05.774Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"80f2984e-d83f-4153-af99-761bd0100398"}
{"level":"info","ts":"2026-11-03T15:11:05.774Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"80f2984e-d83f-4153-af99-761bd0100398"}
{"level":"info","ts":"2026-11-03T15:11:05.774Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"80f2984e-d83f-4153-af99-761bd0100398","1 head service found":"counter-auth-kk8bt-head-svc"}
{"level":"info","ts":"2026-11-03T15:11:05.774Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"80f2984e-d83f-4153-af99-761bd0100398","Found 1 head Pod":"counter-auth-kk8bt-head-27gn8","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:11:05.774Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"80f2984e-d83f-4153-af99-761bd0100398","head Pod":"counter-auth-kk8bt-head-27gn8","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-kk8bt-head-27gn8. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:11:05.774Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"80f2984e-d83f-4153-af99-761bd0100398","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:11:05.774Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"80f2984e-d83f-4153-af99-761bd0100398","worker Pod":"counter-auth-kk8bt-small-group-worker-k56zq","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-kk8bt-small-group-worker-k56zq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:11:05.774Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"80f2984e-d83f-4153-af99-761bd0100398","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:11:05.774Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"80f2984e-d83f-4153-af99-761bd0100398","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:11:05.774Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"80f2984e-d83f-4153-af99-761bd0100398","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:11:05.774Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"80f2984e-d83f-4153-af99-761bd0100398","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:11:05.775Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"80f2984e-d83f-4153-af99-761bd0100398","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:11:05.775Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"80f2984e-d83f-4153-af99-761bd0100398","seconds":300}
{"level":"info","ts":"2026-11-03T15:11:07.799Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"8e146eb2-00b2-4470-901d-eb9d0710693a","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:11:07.799Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"8e146eb2-00b2-4470-901d-eb9d0710693a","name":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:11:07.800Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"8e146eb2-00b2-4470-901d-eb9d0710693a","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:11:07.800Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"8e146eb2-00b2-4470-901d-eb9d0710693a"}
{"level":"info","ts":"2026-11-03T15:11:07.800Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"8e146eb2-00b2-4470-901d-eb9d0710693a","rayCluster":"counter-auth-kk8bt","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:11:07.805Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"8e146eb2-00b2-4470-901d-eb9d0710693a","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:11:04Z"},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","dashboardURL":"counter-auth-kk8bt-head-svc.test-ns-mvz6h.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:03Z","stateTransitionTimes":{"ready":"2026-11-03T15:11:03Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:10:31Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:11:03Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:11:04Z"},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","dashboardURL":"counter-auth-kk8bt-head-svc.test-ns-mvz6h.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:03Z","stateTransitionTimes":{"ready":"2026-11-03T15:11:03Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:10:31Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:11:03Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:11:07.805Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"8e146eb2-00b2-4470-901d-eb9d0710693a","old JobStatus":"PENDING","new JobStatus":"RUNNING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-11-03T15:11:07.920Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"a70b0a0e-0392-4e5a-9818-f189996916ca","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:11:07.920Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"a70b0a0e-0392-4e5a-9818-f189996916ca","name":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:11:07.920Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"a70b0a0e-0392-4e5a-9818-f189996916ca","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:11:07.920Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"a70b0a0e-0392-4e5a-9818-f189996916ca"}
{"level":"info","ts":"2026-11-03T15:11:07.920Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"a70b0a0e-0392-4e5a-9818-f189996916ca","rayCluster":"counter-auth-kk8bt","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:11:07.923Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"a70b0a0e-0392-4e5a-9818-f189996916ca","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:11:04Z"},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","dashboardURL":"counter-auth-kk8bt-head-svc.test-ns-mvz6h.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:03Z","stateTransitionTimes":{"ready":"2026-11-03T15:11:03Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:10:31Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:11:03Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:11:04Z"},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","dashboardURL":"counter-auth-kk8bt-head-svc.test-ns-mvz6h.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:03Z","stateTransitionTimes":{"ready":"2026-11-03T15:11:03Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:10:31Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:11:03Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:11:10.923Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"ade06da9-d95e-415d-9b4b-d66e98757df2","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:11:10.923Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"ade06da9-d95e-415d-9b4b-d66e98757df2","name":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:11:10.923Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"ade06da9-d95e-415d-9b4b-d66e98757df2","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:11:10.924Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"ade06da9-d95e-415d-9b4b-d66e98757df2"}
{"level":"info","ts":"2026-11-03T15:11:10.924Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"ade06da9-d95e-415d-9b4b-d66e98757df2","rayCluster":"counter-auth-kk8bt","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:11:10.927Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"ade06da9-d95e-415d-9b4b-d66e98757df2","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:11:04Z"},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","dashboardURL":"counter-auth-kk8bt-head-svc.test-ns-mvz6h.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:09:09Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:03Z","stateTransitionTimes":{"ready":"2026-11-03T15:11:03Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:10:31Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:11:03Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:11:04Z","endTime":"2026-11-03T15:11:08Z"},"jobId":"counter-auth-lw9mj","rayClusterName":"counter-auth-kk8bt","dashboardURL":"counter-auth-kk8bt-head-svc.test-ns-mvz6h.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Complete","message":"Job finished successfully.","startTime":"2026-11-03T15:09:09Z","succeeded":1,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:03Z","stateTransitionTimes":{"ready":"2026-11-03T15:11:03Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.34","serviceIP":"10.132.0.34","podName":"counter-auth-kk8bt-head-27gn8","serviceName":"counter-auth-kk8bt-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:10:31Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:11:03Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:09:10Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:09:10Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:11:10.927Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"ade06da9-d95e-415d-9b4b-d66e98757df2","old JobStatus":"RUNNING","new JobStatus":"SUCCEEDED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Complete"}
{"level":"info","ts":"2026-11-03T15:11:11.024Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"82dcd097-ac6d-4f77-b9a9-9f9c74e50e76","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:11:11.024Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"82dcd097-ac6d-4f77-b9a9-9f9c74e50e76","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Complete","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-11-03 15:11:10 +0000 UTC","Now":"2026-11-03T15:11:11.024Z","ShutdownTime":"2026-11-03T15:11:10.000Z"}
{"level":"info","ts":"2026-11-03T15:11:11.083Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob is deleted","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"82dcd097-ac6d-4f77-b9a9-9f9c74e50e76","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:11:11.083Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"82dcd097-ac6d-4f77-b9a9-9f9c74e50e76","isClusterDeleted":false}
{"level":"info","ts":"2026-11-03T15:11:11.083Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"82dcd097-ac6d-4f77-b9a9-9f9c74e50e76","deletionMechanism":"ShutdownAfterJobFinishes","RayCluster":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:11:11.083Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"reconcileID":"4d1177d9-22a0-4200-8d62-65ff9358659f"}
{"level":"info","ts":"2026-11-03T15:11:11.083Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-auth-kk8bt","reconcileID":"4aa81fc9-748c-4176-b24f-256252b2677e","namespacedName":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:11:11.083Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-auth-kk8bt","reconcileID":"4aa81fc9-748c-4176-b24f-256252b2677e","cluster":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:11:11.083Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-auth-kk8bt","reconcileID":"4aa81fc9-748c-4176-b24f-256252b2677e","cluster":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:11:11.083Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"c28bac2a-4796-440a-8329-8e64946fc391","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:11:11.083Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-auth-kk8bt","reconcileID":"4aa81fc9-748c-4176-b24f-256252b2677e","namespace":"test-ns-mvz6h","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-11-03T15:11:11.083Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"c28bac2a-4796-440a-8329-8e64946fc391","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Complete","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-11-03 15:11:10 +0000 UTC","Now":"2026-11-03T15:11:11.083Z","ShutdownTime":"2026-11-03T15:11:10.000Z"}
{"level":"info","ts":"2026-11-03T15:11:11.083Z","logger":"controllers.RayJob","msg":"The deletion of the associated RayCluster for RayJob is ongoing.","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"c28bac2a-4796-440a-8329-8e64946fc391","RayCluster":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:11:11.084Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"c28bac2a-4796-440a-8329-8e64946fc391","isClusterDeleted":false}
{"level":"info","ts":"2026-11-03T15:11:11.084Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"c28bac2a-4796-440a-8329-8e64946fc391","deletionMechanism":"ShutdownAfterJobFinishes","RayCluster":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:11:11.135Z","logger":"authentication-controller","msg":"Deleted shared ReferenceGrant (last cluster with auth)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-auth-kk8bt","reconcileID":"4aa81fc9-748c-4176-b24f-256252b2677e","name":"kuberay-gateway-access","namespace":"test-ns-mvz6h"}
{"level":"info","ts":"2026-11-03T15:11:11.188Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-auth-kk8bt","reconcileID":"4aa81fc9-748c-4176-b24f-256252b2677e","name":"test-ns-mvz6h-counter-auth-kk8bt","namespace":"opendatahub"}
{"level":"info","ts":"2026-11-03T15:11:11.240Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-auth-kk8bt","reconcileID":"4aa81fc9-748c-4176-b24f-256252b2677e","configMap":"kube-rbac-proxy-config-counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:11:11.294Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-auth-kk8bt","reconcileID":"4aa81fc9-748c-4176-b24f-256252b2677e","serviceAccount":"counter-auth-kk8bt-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-11-03T15:11:11.478Z","logger":"raycluster-resource","msg":"default","name":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:11:11.478Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"}
{"level":"info","ts":"2026-11-03T15:11:11.641Z","logger":"raycluster-resource","msg":"validate update","name":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:11:11.695Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-auth-kk8bt","reconcileID":"4aa81fc9-748c-4176-b24f-256252b2677e","cluster":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:11:11.696Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-auth-kk8bt","reconcileID":"523d32a2-64b1-404a-aa53-5fe903935370","namespacedName":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:11:11.696Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-auth-kk8bt","reconcileID":"523d32a2-64b1-404a-aa53-5fe903935370","cluster":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:11:11.696Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-auth-kk8bt","reconcileID":"523d32a2-64b1-404a-aa53-5fe903935370","cluster":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:11:11.696Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-auth-kk8bt","reconcileID":"523d32a2-64b1-404a-aa53-5fe903935370","namespace":"test-ns-mvz6h","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-11-03T15:11:11.696Z","logger":"authentication-controller","msg":"ReferenceGrant already deleted","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-auth-kk8bt","reconcileID":"523d32a2-64b1-404a-aa53-5fe903935370","name":"kuberay-gateway-access"}
{"level":"info","ts":"2026-11-03T15:11:11.696Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"29ce7438-37e9-40a2-b640-2161a811b04c","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:11:11.696Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"29ce7438-37e9-40a2-b640-2161a811b04c","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Complete","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-11-03 15:11:10 +0000 UTC","Now":"2026-11-03T15:11:11.696Z","ShutdownTime":"2026-11-03T15:11:10.000Z"}
{"level":"info","ts":"2026-11-03T15:11:11.696Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"29ce7438-37e9-40a2-b640-2161a811b04c","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:11:11.696Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"29ce7438-37e9-40a2-b640-2161a811b04c","isClusterDeleted":true}
{"level":"info","ts":"2026-11-03T15:11:11.696Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"29ce7438-37e9-40a2-b640-2161a811b04c","deletionMechanism":"ShutdownAfterJobFinishes","RayCluster":"counter-auth-kk8bt"}
{"level":"error","ts":"2026-11-03T15:11:11.753Z","logger":"authentication-controller","msg":"Failed to remove finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-auth-kk8bt","reconcileID":"523d32a2-64b1-404a-aa53-5fe903935370","error":"rayclusters.ray.io \"counter-auth-kk8bt\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleDeletion\n\t/workspace/controllers/ray/authentication_controller.go:195\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:11:11.753Z","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-auth-kk8bt","reconcileID":"523d32a2-64b1-404a-aa53-5fe903935370"}
{"level":"error","ts":"2026-11-03T15:11:11.753Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-auth-kk8bt","reconcileID":"523d32a2-64b1-404a-aa53-5fe903935370","error":"rayclusters.ray.io \"counter-auth-kk8bt\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:11:11.753Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-auth-kk8bt","reconcileID":"37e61824-c360-4335-84e8-fe07d5367064","namespacedName":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:11:11.753Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-auth-kk8bt","reconcileID":"37e61824-c360-4335-84e8-fe07d5367064"}
{"level":"info","ts":"2026-11-03T15:11:11.753Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-auth-kk8bt","reconcileID":"37e61824-c360-4335-84e8-fe07d5367064","namespace":"test-ns-mvz6h","authClusters":0}
{"level":"info","ts":"2026-11-03T15:11:11.753Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"},"namespace":"test-ns-mvz6h","name":"counter-auth-kk8bt","reconcileID":"37e61824-c360-4335-84e8-fe07d5367064","namespace":"test-ns-mvz6h"}
{"level":"info","ts":"2026-11-03T15:11:12.367Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"119c7a65-867c-4388-95ee-ca4fd102bbd7","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:11:12.367Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"119c7a65-867c-4388-95ee-ca4fd102bbd7","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-11-03T15:11:12.425Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"119c7a65-867c-4388-95ee-ca4fd102bbd7"}
{"level":"info","ts":"2026-11-03T15:11:12.425Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"119c7a65-867c-4388-95ee-ca4fd102bbd7","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-11-03T15:11:12.426Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"119c7a65-867c-4388-95ee-ca4fd102bbd7","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:11:12.426Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"119c7a65-867c-4388-95ee-ca4fd102bbd7","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:11:12.484Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"da27eedc-1c10-4cd0-81c6-9d166b3b4b3e","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:11:12.484Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"da27eedc-1c10-4cd0-81c6-9d166b3b4b3e"}
{"level":"info","ts":"2026-11-03T15:11:12.484Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"da27eedc-1c10-4cd0-81c6-9d166b3b4b3e","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-11-03T15:11:12.484Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"da27eedc-1c10-4cd0-81c6-9d166b3b4b3e","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-7j5wt","rayClusterName":"long-running-vmfhf","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:11:12.484Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"da27eedc-1c10-4cd0-81c6-9d166b3b4b3e","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:11:12.537Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"da27eedc-1c10-4cd0-81c6-9d166b3b4b3e","error":"Operation cannot be fulfilled on rayjobs.ray.io \"long-running\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-11-03T15:11:12.537Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"da27eedc-1c10-4cd0-81c6-9d166b3b4b3e"}
{"level":"error","ts":"2026-11-03T15:11:12.537Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"da27eedc-1c10-4cd0-81c6-9d166b3b4b3e","error":"Operation cannot be fulfilled on rayjobs.ray.io \"long-running\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:11:12.537Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"291aced0-df7f-42a2-aebc-488000ed7fb3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:11:12.537Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"291aced0-df7f-42a2-aebc-488000ed7fb3","name":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:11:12.537Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"291aced0-df7f-42a2-aebc-488000ed7fb3","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:11:12.537Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"291aced0-df7f-42a2-aebc-488000ed7fb3","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:11:12.588Z","logger":"raycluster-resource","msg":"default","name":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:11:12.588Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"long-running-2zj8b","namespace":"test-ns-5lqjm"}
{"level":"info","ts":"2026-11-03T15:11:12.642Z","logger":"raycluster-resource","msg":"validate create","name":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:11:12.696Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"291aced0-df7f-42a2-aebc-488000ed7fb3","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:11:12.696Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"291aced0-df7f-42a2-aebc-488000ed7fb3"}
{"level":"info","ts":"2026-11-03T15:11:12.696Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"291aced0-df7f-42a2-aebc-488000ed7fb3","RayCluster":"long-running-2zj8b","State":""}
{"level":"info","ts":"2026-11-03T15:11:12.696Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"291aced0-df7f-42a2-aebc-488000ed7fb3","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:11:12.696Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"6f9f8903-ec36-41f0-856d-a09a3f8ba021"}
{"level":"info","ts":"2026-11-03T15:11:12.696Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"6f9f8903-ec36-41f0-856d-a09a3f8ba021"}
{"level":"info","ts":"2026-11-03T15:11:12.696Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"49048269-72c3-4a17-80f6-b816956c732f","cluster":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:11:12.696Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"663d4b7f-f17b-40d9-91be-30e418b1372f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:11:12.696Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"663d4b7f-f17b-40d9-91be-30e418b1372f","name":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:11:12.696Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"663d4b7f-f17b-40d9-91be-30e418b1372f","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:11:12.696Z","logger":"controllers.NetworkPolicy","msg":"Reconciling NetworkPolicies for RayCluster","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"c69dbb1e-5d6f-4cb3-a04e-62e9df13d3b3","cluster":"long-running-2zj8b","namespace":"test-ns-5lqjm"}
{"level":"info","ts":"2026-11-03T15:11:12.697Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"663d4b7f-f17b-40d9-91be-30e418b1372f"}
{"level":"info","ts":"2026-11-03T15:11:12.697Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"663d4b7f-f17b-40d9-91be-30e418b1372f","RayCluster":"long-running-2zj8b","State":""}
{"level":"info","ts":"2026-11-03T15:11:12.697Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"663d4b7f-f17b-40d9-91be-30e418b1372f","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:11:12.697Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"long-running-2zj8b","reconcileID":"87db139d-d6dd-43d6-a2fa-eae8e8abf993","namespacedName":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:11:12.697Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"long-running-2zj8b","reconcileID":"87db139d-d6dd-43d6-a2fa-eae8e8abf993","mode":"IntegratedOAuth","cluster":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:11:12.697Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"long-running-2zj8b","reconcileID":"87db139d-d6dd-43d6-a2fa-eae8e8abf993","cluster":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:11:12.697Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"long-running-2zj8b","reconcileID":"87db139d-d6dd-43d6-a2fa-eae8e8abf993","cluster":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:11:12.749Z","logger":"raycluster-resource","msg":"default","name":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:11:12.749Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"long-running-2zj8b","namespace":"test-ns-5lqjm"}
{"level":"info","ts":"2026-11-03T15:11:12.752Z","logger":"controllers.NetworkPolicy","msg":"Successfully created NetworkPolicy","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"c69dbb1e-5d6f-4cb3-a04e-62e9df13d3b3","name":"long-running-2zj8b-head"}
{"level":"info","ts":"2026-11-03T15:11:12.753Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"6f9f8903-ec36-41f0-856d-a09a3f8ba021","name":"long-running-2zj8b-head-svc"}
{"level":"info","ts":"2026-11-03T15:11:12.753Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"6f9f8903-ec36-41f0-856d-a09a3f8ba021"}
{"level":"info","ts":"2026-11-03T15:11:12.802Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"6f9f8903-ec36-41f0-856d-a09a3f8ba021","error":"certificate ray-head-cert-long-running-2zj8b not found: certificates.cert-manager.io \"ray-head-cert-long-running-2zj8b\" not found"}
{"level":"error","ts":"2026-11-03T15:11:12.802Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"6f9f8903-ec36-41f0-856d-a09a3f8ba021","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-2zj8b not found: certificates.cert-manager.io \"ray-head-cert-long-running-2zj8b\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:11:12.802Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"6f9f8903-ec36-41f0-856d-a09a3f8ba021","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-2zj8b not found: certificates.cert-manager.io \"ray-head-cert-long-running-2zj8b\" not found"}
{"level":"info","ts":"2026-11-03T15:11:12.802Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"6f9f8903-ec36-41f0-856d-a09a3f8ba021","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"long-running-2zj8b","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:11:12.802Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"6f9f8903-ec36-41f0-856d-a09a3f8ba021","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"long-running-2zj8b","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:11:12.802Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"6f9f8903-ec36-41f0-856d-a09a3f8ba021","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"long-running-2zj8b","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:11:12.802Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"6f9f8903-ec36-41f0-856d-a09a3f8ba021","name":"long-running-2zj8b","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:12Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:11:12.804Z","logger":"raycluster-resource","msg":"validate update","name":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:11:12.807Z","logger":"controllers.NetworkPolicy","msg":"Successfully created NetworkPolicy","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"c69dbb1e-5d6f-4cb3-a04e-62e9df13d3b3","name":"long-running-2zj8b-workers-small-group"}
{"level":"info","ts":"2026-11-03T15:11:12.807Z","logger":"controllers.NetworkPolicy","msg":"Successfully reconciled NetworkPolicies for RayCluster","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"c69dbb1e-5d6f-4cb3-a04e-62e9df13d3b3","cluster":"long-running-2zj8b","namespace":"test-ns-5lqjm"}
{"level":"info","ts":"2026-11-03T15:11:12.808Z","logger":"controllers.NetworkPolicy","msg":"Reconciling NetworkPolicies for RayCluster","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"4e9ab00b-f7be-4bef-b252-8eaf5d4dcac9","cluster":"long-running-2zj8b","namespace":"test-ns-5lqjm"}
{"level":"info","ts":"2026-11-03T15:11:12.858Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"1b37bba8-fa06-46ff-956c-6f5514e7818e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:11:12.858Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"1b37bba8-fa06-46ff-956c-6f5514e7818e","name":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:11:12.858Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"1b37bba8-fa06-46ff-956c-6f5514e7818e","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:11:12.858Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"1b37bba8-fa06-46ff-956c-6f5514e7818e"}
{"level":"info","ts":"2026-11-03T15:11:12.858Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"1b37bba8-fa06-46ff-956c-6f5514e7818e","RayCluster":"long-running-2zj8b","State":""}
{"level":"info","ts":"2026-11-03T15:11:12.858Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"1b37bba8-fa06-46ff-956c-6f5514e7818e","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:11:12.862Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"6f9f8903-ec36-41f0-856d-a09a3f8ba021","name":"long-running-2zj8b","error":"Operation cannot be fulfilled on rayclusters.ray.io \"long-running-2zj8b\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-5lqjm","name":"long-running-2zj8b"}}
{"level":"info","ts":"2026-11-03T15:11:12.862Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"6f9f8903-ec36-41f0-856d-a09a3f8ba021"}
{"level":"error","ts":"2026-11-03T15:11:12.862Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"6f9f8903-ec36-41f0-856d-a09a3f8ba021","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-2zj8b not found: certificates.cert-manager.io \"ray-head-cert-long-running-2zj8b\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:11:12.862Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"6e7d21bf-bf4b-4a55-bcc3-5106a1341b88"}
{"level":"info","ts":"2026-11-03T15:11:12.862Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"6e7d21bf-bf4b-4a55-bcc3-5106a1341b88"}
{"level":"info","ts":"2026-11-03T15:11:12.862Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"6e7d21bf-bf4b-4a55-bcc3-5106a1341b88","1 head service found":"long-running-2zj8b-head-svc"}
{"level":"info","ts":"2026-11-03T15:11:12.862Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"6e7d21bf-bf4b-4a55-bcc3-5106a1341b88"}
{"level":"info","ts":"2026-11-03T15:11:12.912Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"long-running-2zj8b","reconcileID":"87db139d-d6dd-43d6-a2fa-eae8e8abf993","name":"long-running-2zj8b-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-11-03T15:11:12.913Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"6e7d21bf-bf4b-4a55-bcc3-5106a1341b88","error":"certificate ray-head-cert-long-running-2zj8b not found: certificates.cert-manager.io \"ray-head-cert-long-running-2zj8b\" not found"}
{"level":"error","ts":"2026-11-03T15:11:12.913Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"6e7d21bf-bf4b-4a55-bcc3-5106a1341b88","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-2zj8b not found: certificates.cert-manager.io \"ray-head-cert-long-running-2zj8b\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:11:12.913Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"6e7d21bf-bf4b-4a55-bcc3-5106a1341b88","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-2zj8b not found: certificates.cert-manager.io \"ray-head-cert-long-running-2zj8b\" not found"}
{"level":"info","ts":"2026-11-03T15:11:12.913Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"6e7d21bf-bf4b-4a55-bcc3-5106a1341b88","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"long-running-2zj8b","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:11:12.913Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"6e7d21bf-bf4b-4a55-bcc3-5106a1341b88","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"long-running-2zj8b","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:11:12.913Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"6e7d21bf-bf4b-4a55-bcc3-5106a1341b88","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"long-running-2zj8b","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:11:12.913Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"6e7d21bf-bf4b-4a55-bcc3-5106a1341b88","name":"long-running-2zj8b","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:12Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:11:12.917Z","logger":"controllers.NetworkPolicy","msg":"Successfully reconciled NetworkPolicies for RayCluster","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"4e9ab00b-f7be-4bef-b252-8eaf5d4dcac9","cluster":"long-running-2zj8b","namespace":"test-ns-5lqjm"}
{"level":"info","ts":"2026-11-03T15:11:12.965Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"long-running-2zj8b","reconcileID":"87db139d-d6dd-43d6-a2fa-eae8e8abf993","name":"kuberay-gateway-access","namespace":"test-ns-5lqjm","operation":"created"}
{"level":"info","ts":"2026-11-03T15:11:12.972Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"6e7d21bf-bf4b-4a55-bcc3-5106a1341b88"}
{"level":"error","ts":"2026-11-03T15:11:12.972Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"6e7d21bf-bf4b-4a55-bcc3-5106a1341b88","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-2zj8b not found: certificates.cert-manager.io \"ray-head-cert-long-running-2zj8b\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:11:12.972Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"1ddfb388-41ea-454e-82b2-fefa2a893798","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:11:12.972Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"1ddfb388-41ea-454e-82b2-fefa2a893798","name":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:11:12.972Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"1ddfb388-41ea-454e-82b2-fefa2a893798","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:11:12.972Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"1ddfb388-41ea-454e-82b2-fefa2a893798"}
{"level":"info","ts":"2026-11-03T15:11:12.972Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"1ddfb388-41ea-454e-82b2-fefa2a893798","RayCluster":"long-running-2zj8b","State":""}
{"level":"info","ts":"2026-11-03T15:11:12.972Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"1ddfb388-41ea-454e-82b2-fefa2a893798","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:12Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:11:12.972Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"1ddfb388-41ea-454e-82b2-fefa2a893798","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:11:12.982Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"12efbff9-fbe9-4bc9-b323-ecc4247ce411"}
{"level":"info","ts":"2026-11-03T15:11:12.982Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"12efbff9-fbe9-4bc9-b323-ecc4247ce411"}
{"level":"info","ts":"2026-11-03T15:11:12.982Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"12efbff9-fbe9-4bc9-b323-ecc4247ce411","1 head service found":"long-running-2zj8b-head-svc"}
{"level":"info","ts":"2026-11-03T15:11:12.982Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"12efbff9-fbe9-4bc9-b323-ecc4247ce411"}
{"level":"info","ts":"2026-11-03T15:11:13.019Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"long-running-2zj8b","reconcileID":"87db139d-d6dd-43d6-a2fa-eae8e8abf993","name":"test-ns-5lqjm-long-running-2zj8b","operation":"created"}
{"level":"info","ts":"2026-11-03T15:11:13.032Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"12efbff9-fbe9-4bc9-b323-ecc4247ce411","error":"certificate ray-head-cert-long-running-2zj8b not found: certificates.cert-manager.io \"ray-head-cert-long-running-2zj8b\" not found"}
{"level":"error","ts":"2026-11-03T15:11:13.032Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"12efbff9-fbe9-4bc9-b323-ecc4247ce411","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-2zj8b not found: certificates.cert-manager.io \"ray-head-cert-long-running-2zj8b\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:11:13.032Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"12efbff9-fbe9-4bc9-b323-ecc4247ce411","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-2zj8b not found: certificates.cert-manager.io \"ray-head-cert-long-running-2zj8b\" not found"}
{"level":"info","ts":"2026-11-03T15:11:13.032Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"12efbff9-fbe9-4bc9-b323-ecc4247ce411","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"long-running-2zj8b","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:11:13.032Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"12efbff9-fbe9-4bc9-b323-ecc4247ce411","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"long-running-2zj8b","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:11:13.032Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"12efbff9-fbe9-4bc9-b323-ecc4247ce411","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"long-running-2zj8b","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:11:13.032Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"12efbff9-fbe9-4bc9-b323-ecc4247ce411"}
{"level":"error","ts":"2026-11-03T15:11:13.032Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"12efbff9-fbe9-4bc9-b323-ecc4247ce411","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-2zj8b not found: certificates.cert-manager.io \"ray-head-cert-long-running-2zj8b\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:11:13.032Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"eadeb89a-8895-488a-8422-165e3ed4b332","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:11:13.032Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"eadeb89a-8895-488a-8422-165e3ed4b332","name":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:11:13.032Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"eadeb89a-8895-488a-8422-165e3ed4b332","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:11:13.032Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"eadeb89a-8895-488a-8422-165e3ed4b332"}
{"level":"info","ts":"2026-11-03T15:11:13.032Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"eadeb89a-8895-488a-8422-165e3ed4b332","RayCluster":"long-running-2zj8b","State":""}
{"level":"info","ts":"2026-11-03T15:11:13.032Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"eadeb89a-8895-488a-8422-165e3ed4b332","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:12Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:12Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:11:13.052Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"e945d408-8d4b-4bc1-9eb4-919ecb8e8923"}
{"level":"info","ts":"2026-11-03T15:11:13.052Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"e945d408-8d4b-4bc1-9eb4-919ecb8e8923"}
{"level":"info","ts":"2026-11-03T15:11:13.052Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"e945d408-8d4b-4bc1-9eb4-919ecb8e8923","1 head service found":"long-running-2zj8b-head-svc"}
{"level":"info","ts":"2026-11-03T15:11:13.052Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"e945d408-8d4b-4bc1-9eb4-919ecb8e8923"}
{"level":"info","ts":"2026-11-03T15:11:13.071Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"long-running-2zj8b","reconcileID":"87db139d-d6dd-43d6-a2fa-eae8e8abf993","name":"kube-rbac-proxy-config-long-running-2zj8b","operation":"created"}
{"level":"info","ts":"2026-11-03T15:11:13.104Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"e945d408-8d4b-4bc1-9eb4-919ecb8e8923","error":"certificate ray-head-cert-long-running-2zj8b not found: certificates.cert-manager.io \"ray-head-cert-long-running-2zj8b\" not found"}
{"level":"error","ts":"2026-11-03T15:11:13.104Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"e945d408-8d4b-4bc1-9eb4-919ecb8e8923","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-2zj8b not found: certificates.cert-manager.io \"ray-head-cert-long-running-2zj8b\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:11:13.104Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"e945d408-8d4b-4bc1-9eb4-919ecb8e8923","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-2zj8b not found: certificates.cert-manager.io \"ray-head-cert-long-running-2zj8b\" not found"}
{"level":"info","ts":"2026-11-03T15:11:13.104Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"e945d408-8d4b-4bc1-9eb4-919ecb8e8923","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"long-running-2zj8b","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:11:13.104Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"e945d408-8d4b-4bc1-9eb4-919ecb8e8923","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"long-running-2zj8b","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:11:13.104Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"e945d408-8d4b-4bc1-9eb4-919ecb8e8923","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"long-running-2zj8b","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:11:13.104Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"e945d408-8d4b-4bc1-9eb4-919ecb8e8923"}
{"level":"error","ts":"2026-11-03T15:11:13.104Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"e945d408-8d4b-4bc1-9eb4-919ecb8e8923","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-2zj8b not found: certificates.cert-manager.io \"ray-head-cert-long-running-2zj8b\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-03T15:11:13.125Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"long-running-2zj8b","reconcileID":"87db139d-d6dd-43d6-a2fa-eae8e8abf993","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"long-running-2zj8b\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:167\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:11:13.125Z","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"long-running-2zj8b","reconcileID":"87db139d-d6dd-43d6-a2fa-eae8e8abf993"}
{"level":"error","ts":"2026-11-03T15:11:13.125Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"long-running-2zj8b","reconcileID":"87db139d-d6dd-43d6-a2fa-eae8e8abf993","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"long-running-2zj8b\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:11:13.125Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"long-running-2zj8b","reconcileID":"2cfe595c-afd4-4da8-b486-bb700561b781","namespacedName":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:11:13.125Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"long-running-2zj8b","reconcileID":"2cfe595c-afd4-4da8-b486-bb700561b781","mode":"IntegratedOAuth","cluster":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:11:13.125Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"long-running-2zj8b","reconcileID":"2cfe595c-afd4-4da8-b486-bb700561b781","cluster":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:11:13.145Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"d8096fcc-bbe8-4358-a70b-d2d63051aa6a"}
{"level":"info","ts":"2026-11-03T15:11:13.145Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"d8096fcc-bbe8-4358-a70b-d2d63051aa6a"}
{"level":"info","ts":"2026-11-03T15:11:13.145Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"d8096fcc-bbe8-4358-a70b-d2d63051aa6a","1 head service found":"long-running-2zj8b-head-svc"}
{"level":"info","ts":"2026-11-03T15:11:13.145Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"d8096fcc-bbe8-4358-a70b-d2d63051aa6a"}
{"level":"info","ts":"2026-11-03T15:11:13.185Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"long-running-2zj8b","reconcileID":"2cfe595c-afd4-4da8-b486-bb700561b781","cluster":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:11:13.186Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"b094eb67-e57b-4d11-921c-909c7b707fd0","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:11:13.186Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"b094eb67-e57b-4d11-921c-909c7b707fd0","name":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:11:13.186Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"b094eb67-e57b-4d11-921c-909c7b707fd0","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:11:13.186Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"b094eb67-e57b-4d11-921c-909c7b707fd0"}
{"level":"info","ts":"2026-11-03T15:11:13.186Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"b094eb67-e57b-4d11-921c-909c7b707fd0","RayCluster":"long-running-2zj8b","State":""}
{"level":"info","ts":"2026-11-03T15:11:13.186Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"b094eb67-e57b-4d11-921c-909c7b707fd0","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:12Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:12Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:11:13.186Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"b094eb67-e57b-4d11-921c-909c7b707fd0","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:11:13.194Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"d8096fcc-bbe8-4358-a70b-d2d63051aa6a","error":"certificate ray-head-cert-long-running-2zj8b not found: certificates.cert-manager.io \"ray-head-cert-long-running-2zj8b\" not found"}
{"level":"error","ts":"2026-11-03T15:11:13.194Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"d8096fcc-bbe8-4358-a70b-d2d63051aa6a","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-2zj8b not found: certificates.cert-manager.io \"ray-head-cert-long-running-2zj8b\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:11:13.194Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"d8096fcc-bbe8-4358-a70b-d2d63051aa6a","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-2zj8b not found: certificates.cert-manager.io \"ray-head-cert-long-running-2zj8b\" not found"}
{"level":"info","ts":"2026-11-03T15:11:13.194Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"d8096fcc-bbe8-4358-a70b-d2d63051aa6a","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"long-running-2zj8b","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:11:13.194Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"d8096fcc-bbe8-4358-a70b-d2d63051aa6a","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"long-running-2zj8b","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:11:13.194Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"d8096fcc-bbe8-4358-a70b-d2d63051aa6a","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"long-running-2zj8b","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:11:13.194Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"d8096fcc-bbe8-4358-a70b-d2d63051aa6a"}
{"level":"error","ts":"2026-11-03T15:11:13.194Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"d8096fcc-bbe8-4358-a70b-d2d63051aa6a","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-2zj8b not found: certificates.cert-manager.io \"ray-head-cert-long-running-2zj8b\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:11:13.246Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"f4d65c04-2c56-4835-82aa-8be24f522161","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:11:13.246Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"f4d65c04-2c56-4835-82aa-8be24f522161","name":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:11:13.246Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"f4d65c04-2c56-4835-82aa-8be24f522161","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:11:13.246Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"f4d65c04-2c56-4835-82aa-8be24f522161"}
{"level":"info","ts":"2026-11-03T15:11:13.246Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"f4d65c04-2c56-4835-82aa-8be24f522161","RayCluster":"long-running-2zj8b","State":""}
{"level":"info","ts":"2026-11-03T15:11:13.246Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"f4d65c04-2c56-4835-82aa-8be24f522161","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:12Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:12Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:11:13.275Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"8f9d3bf9-a98c-47d4-9e64-25f76fd764d0"}
{"level":"info","ts":"2026-11-03T15:11:13.275Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"8f9d3bf9-a98c-47d4-9e64-25f76fd764d0"}
{"level":"info","ts":"2026-11-03T15:11:13.275Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"8f9d3bf9-a98c-47d4-9e64-25f76fd764d0","1 head service found":"long-running-2zj8b-head-svc"}
{"level":"info","ts":"2026-11-03T15:11:13.275Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"8f9d3bf9-a98c-47d4-9e64-25f76fd764d0"}
{"level":"info","ts":"2026-11-03T15:11:13.324Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"8f9d3bf9-a98c-47d4-9e64-25f76fd764d0","error":"certificate ray-head-cert-long-running-2zj8b not found: certificates.cert-manager.io \"ray-head-cert-long-running-2zj8b\" not found"}
{"level":"error","ts":"2026-11-03T15:11:13.324Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"8f9d3bf9-a98c-47d4-9e64-25f76fd764d0","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-2zj8b not found: certificates.cert-manager.io \"ray-head-cert-long-running-2zj8b\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:11:13.324Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"8f9d3bf9-a98c-47d4-9e64-25f76fd764d0","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-2zj8b not found: certificates.cert-manager.io \"ray-head-cert-long-running-2zj8b\" not found"}
{"level":"info","ts":"2026-11-03T15:11:13.324Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"8f9d3bf9-a98c-47d4-9e64-25f76fd764d0","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"long-running-2zj8b","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:11:13.324Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"8f9d3bf9-a98c-47d4-9e64-25f76fd764d0","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"long-running-2zj8b","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:11:13.324Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"8f9d3bf9-a98c-47d4-9e64-25f76fd764d0","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"long-running-2zj8b","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:11:13.324Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"8f9d3bf9-a98c-47d4-9e64-25f76fd764d0"}
{"level":"error","ts":"2026-11-03T15:11:13.324Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"8f9d3bf9-a98c-47d4-9e64-25f76fd764d0","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-2zj8b not found: certificates.cert-manager.io \"ray-head-cert-long-running-2zj8b\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:11:13.485Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"86d57e83-32bd-4969-977c-3891e728bef7"}
{"level":"info","ts":"2026-11-03T15:11:13.485Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"86d57e83-32bd-4969-977c-3891e728bef7"}
{"level":"info","ts":"2026-11-03T15:11:13.485Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"86d57e83-32bd-4969-977c-3891e728bef7","1 head service found":"long-running-2zj8b-head-svc"}
{"level":"info","ts":"2026-11-03T15:11:13.485Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"86d57e83-32bd-4969-977c-3891e728bef7"}
{"level":"info","ts":"2026-11-03T15:11:13.535Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"86d57e83-32bd-4969-977c-3891e728bef7","error":"certificate ray-head-cert-long-running-2zj8b is not ready at current generation 1"}
{"level":"error","ts":"2026-11-03T15:11:13.535Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"86d57e83-32bd-4969-977c-3891e728bef7","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-2zj8b is not ready at current generation 1","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:11:13.535Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"86d57e83-32bd-4969-977c-3891e728bef7","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-2zj8b is not ready at current generation 1"}
{"level":"info","ts":"2026-11-03T15:11:13.535Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"86d57e83-32bd-4969-977c-3891e728bef7","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"long-running-2zj8b","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:11:13.535Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"86d57e83-32bd-4969-977c-3891e728bef7","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"long-running-2zj8b","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:11:13.535Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"86d57e83-32bd-4969-977c-3891e728bef7","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"long-running-2zj8b","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:11:13.535Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"86d57e83-32bd-4969-977c-3891e728bef7"}
{"level":"error","ts":"2026-11-03T15:11:13.535Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"86d57e83-32bd-4969-977c-3891e728bef7","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-2zj8b is not ready at current generation 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:11:13.635Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"49048269-72c3-4a17-80f6-b816956c732f","certificate":"ray-head-cert-long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:11:13.635Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"49048269-72c3-4a17-80f6-b816956c732f"}
{"level":"info","ts":"2026-11-03T15:11:13.635Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"3ee6fc97-00e1-4bbc-b2c9-b8e8bbf193b7","cluster":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:11:13.856Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"7591fa63-d493-46fb-ae09-61d1e51af420"}
{"level":"info","ts":"2026-11-03T15:11:13.856Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"7591fa63-d493-46fb-ae09-61d1e51af420"}
{"level":"info","ts":"2026-11-03T15:11:13.856Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"7591fa63-d493-46fb-ae09-61d1e51af420","1 head service found":"long-running-2zj8b-head-svc"}
{"level":"info","ts":"2026-11-03T15:11:13.856Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"7591fa63-d493-46fb-ae09-61d1e51af420"}
{"level":"info","ts":"2026-11-03T15:11:13.906Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"7591fa63-d493-46fb-ae09-61d1e51af420","error":"certificate ray-head-cert-long-running-2zj8b is not ready at current generation 1"}
{"level":"error","ts":"2026-11-03T15:11:13.906Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"7591fa63-d493-46fb-ae09-61d1e51af420","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-2zj8b is not ready at current generation 1","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:11:13.907Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"7591fa63-d493-46fb-ae09-61d1e51af420","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-2zj8b is not ready at current generation 1"}
{"level":"info","ts":"2026-11-03T15:11:13.907Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"7591fa63-d493-46fb-ae09-61d1e51af420","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"long-running-2zj8b","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:11:13.907Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"7591fa63-d493-46fb-ae09-61d1e51af420","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"long-running-2zj8b","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:11:13.907Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"7591fa63-d493-46fb-ae09-61d1e51af420","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"long-running-2zj8b","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:11:13.907Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"7591fa63-d493-46fb-ae09-61d1e51af420"}
{"level":"error","ts":"2026-11-03T15:11:13.907Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"7591fa63-d493-46fb-ae09-61d1e51af420","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-2zj8b is not ready at current generation 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:11:14.094Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"3ee6fc97-00e1-4bbc-b2c9-b8e8bbf193b7","certificate":"ray-head-cert-long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:11:14.094Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"3ee6fc97-00e1-4bbc-b2c9-b8e8bbf193b7"}
{"level":"info","ts":"2026-11-03T15:11:14.548Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"388e7d9c-b23a-4758-9a0a-d00edc07ab8c"}
{"level":"info","ts":"2026-11-03T15:11:14.548Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"388e7d9c-b23a-4758-9a0a-d00edc07ab8c"}
{"level":"info","ts":"2026-11-03T15:11:14.548Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"388e7d9c-b23a-4758-9a0a-d00edc07ab8c","1 head service found":"long-running-2zj8b-head-svc"}
{"level":"info","ts":"2026-11-03T15:11:14.548Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"388e7d9c-b23a-4758-9a0a-d00edc07ab8c"}
{"level":"info","ts":"2026-11-03T15:11:14.598Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"388e7d9c-b23a-4758-9a0a-d00edc07ab8c","error":"certificate ray-head-cert-long-running-2zj8b is not ready at current generation 1"}
{"level":"error","ts":"2026-11-03T15:11:14.598Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"388e7d9c-b23a-4758-9a0a-d00edc07ab8c","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-2zj8b is not ready at current generation 1","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:11:14.598Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"388e7d9c-b23a-4758-9a0a-d00edc07ab8c","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-2zj8b is not ready at current generation 1"}
{"level":"info","ts":"2026-11-03T15:11:14.598Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"388e7d9c-b23a-4758-9a0a-d00edc07ab8c","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"long-running-2zj8b","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:11:14.598Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"388e7d9c-b23a-4758-9a0a-d00edc07ab8c","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"long-running-2zj8b","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:11:14.598Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"388e7d9c-b23a-4758-9a0a-d00edc07ab8c","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"long-running-2zj8b","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:11:14.598Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"388e7d9c-b23a-4758-9a0a-d00edc07ab8c"}
{"level":"error","ts":"2026-11-03T15:11:14.598Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"388e7d9c-b23a-4758-9a0a-d00edc07ab8c","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-2zj8b is not ready at current generation 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:11:15.879Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"9b2c17a6-0c0c-460c-8a28-380d9d4af302"}
{"level":"info","ts":"2026-11-03T15:11:15.879Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"9b2c17a6-0c0c-460c-8a28-380d9d4af302"}
{"level":"info","ts":"2026-11-03T15:11:15.879Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"9b2c17a6-0c0c-460c-8a28-380d9d4af302","1 head service found":"long-running-2zj8b-head-svc"}
{"level":"info","ts":"2026-11-03T15:11:15.880Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"9b2c17a6-0c0c-460c-8a28-380d9d4af302"}
{"level":"info","ts":"2026-11-03T15:11:15.930Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"9b2c17a6-0c0c-460c-8a28-380d9d4af302","error":"certificate ray-head-cert-long-running-2zj8b is not ready at current generation 1"}
{"level":"error","ts":"2026-11-03T15:11:15.930Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"9b2c17a6-0c0c-460c-8a28-380d9d4af302","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-2zj8b is not ready at current generation 1","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:11:15.930Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"9b2c17a6-0c0c-460c-8a28-380d9d4af302","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-2zj8b is not ready at current generation 1"}
{"level":"info","ts":"2026-11-03T15:11:15.930Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"9b2c17a6-0c0c-460c-8a28-380d9d4af302","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"long-running-2zj8b","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:11:15.930Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"9b2c17a6-0c0c-460c-8a28-380d9d4af302","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"long-running-2zj8b","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:11:15.930Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"9b2c17a6-0c0c-460c-8a28-380d9d4af302","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"long-running-2zj8b","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:11:15.930Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"9b2c17a6-0c0c-460c-8a28-380d9d4af302"}
{"level":"error","ts":"2026-11-03T15:11:15.930Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"9b2c17a6-0c0c-460c-8a28-380d9d4af302","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-2zj8b is not ready at current generation 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:11:16.247Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"d6f3a44b-a647-426f-808e-fe51075111ff","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:11:16.247Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"d6f3a44b-a647-426f-808e-fe51075111ff","name":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:11:16.247Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"d6f3a44b-a647-426f-808e-fe51075111ff","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:11:16.247Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"d6f3a44b-a647-426f-808e-fe51075111ff"}
{"level":"info","ts":"2026-11-03T15:11:16.247Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"d6f3a44b-a647-426f-808e-fe51075111ff","RayCluster":"long-running-2zj8b","State":""}
{"level":"info","ts":"2026-11-03T15:11:16.247Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"d6f3a44b-a647-426f-808e-fe51075111ff","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:12Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:12Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:11:17.793Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"8304fe1f-2433-4613-ab2c-dd157ba41125","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:11:17.793Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"8304fe1f-2433-4613-ab2c-dd157ba41125","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Complete","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-11-03 15:11:10 +0000 UTC","Now":"2026-11-03T15:11:17.793Z","ShutdownTime":"2026-11-03T15:11:10.000Z"}
{"level":"info","ts":"2026-11-03T15:11:17.793Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"8304fe1f-2433-4613-ab2c-dd157ba41125","RayCluster":{"name":"counter-auth-kk8bt","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:11:17.793Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"8304fe1f-2433-4613-ab2c-dd157ba41125","isClusterDeleted":true}
{"level":"info","ts":"2026-11-03T15:11:17.793Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"8304fe1f-2433-4613-ab2c-dd157ba41125","deletionMechanism":"ShutdownAfterJobFinishes","RayCluster":"counter-auth-kk8bt"}
{"level":"info","ts":"2026-11-03T15:11:17.799Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"6a1447fc-5f08-45e9-a694-0e19c37167c6","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"HTTPMode"}
{"level":"info","ts":"2026-11-03T15:11:17.799Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"6a1447fc-5f08-45e9-a694-0e19c37167c6","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Complete","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-11-03 15:04:57 +0000 UTC","Now":"2026-11-03T15:11:17.799Z","ShutdownTime":"2026-11-03T15:04:57.000Z"}
{"level":"info","ts":"2026-11-03T15:11:17.799Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"6a1447fc-5f08-45e9-a694-0e19c37167c6","RayCluster":{"name":"counter-5gq9h","namespace":"test-ns-mvz6h"}}
{"level":"info","ts":"2026-11-03T15:11:17.799Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"6a1447fc-5f08-45e9-a694-0e19c37167c6","isClusterDeleted":true}
{"level":"info","ts":"2026-11-03T15:11:17.799Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"6a1447fc-5f08-45e9-a694-0e19c37167c6","deletionMechanism":"ShutdownAfterJobFinishes","RayCluster":"counter-5gq9h"}
{"level":"info","ts":"2026-11-03T15:11:18.491Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"42ac6558-4baf-4db0-b8d6-4a8dd117ea6f"}
{"level":"info","ts":"2026-11-03T15:11:18.491Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"42ac6558-4baf-4db0-b8d6-4a8dd117ea6f"}
{"level":"info","ts":"2026-11-03T15:11:18.491Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"42ac6558-4baf-4db0-b8d6-4a8dd117ea6f","1 head service found":"long-running-2zj8b-head-svc"}
{"level":"info","ts":"2026-11-03T15:11:18.491Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"42ac6558-4baf-4db0-b8d6-4a8dd117ea6f"}
{"level":"info","ts":"2026-11-03T15:11:18.592Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"42ac6558-4baf-4db0-b8d6-4a8dd117ea6f","mode":"IntegratedOAuth","cluster":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:11:18.592Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"42ac6558-4baf-4db0-b8d6-4a8dd117ea6f","cluster":"long-running-2zj8b","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-11-03T15:11:18.592Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"42ac6558-4baf-4db0-b8d6-4a8dd117ea6f","cluster":"long-running-2zj8b","authType":"IntegratedOAuth","serviceAccount":"long-running-2zj8b-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-11-03T15:11:18.593Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"42ac6558-4baf-4db0-b8d6-4a8dd117ea6f","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"long-running-2zj8b","ray.io/group":"headgroup","ray.io/identifier":"long-running-2zj8b-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-11-03T15:11:18.593Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"42ac6558-4baf-4db0-b8d6-4a8dd117ea6f","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-11-03T15:11:18.593Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"42ac6558-4baf-4db0-b8d6-4a8dd117ea6f","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-11-03T15:11:18.593Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"42ac6558-4baf-4db0-b8d6-4a8dd117ea6f","rayNodeType":"head","generatedCmd":"ulimit -n ${RAY_START_ULIMIT_OPEN_FILES:-65536}; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-11-03T15:11:18.593Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"42ac6558-4baf-4db0-b8d6-4a8dd117ea6f","enabled":true}
{"level":"info","ts":"2026-11-03T15:11:18.714Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"42ac6558-4baf-4db0-b8d6-4a8dd117ea6f","name":"long-running-2zj8b-head-cklc6"}
{"level":"info","ts":"2026-11-03T15:11:18.714Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"42ac6558-4baf-4db0-b8d6-4a8dd117ea6f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:11:18.714Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"42ac6558-4baf-4db0-b8d6-4a8dd117ea6f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:11:18.714Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"42ac6558-4baf-4db0-b8d6-4a8dd117ea6f","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-11-03T15:11:18.714Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"42ac6558-4baf-4db0-b8d6-4a8dd117ea6f","group":"small-group","indices":{}}
{"level":"info","ts":"2026-11-03T15:11:18.714Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"42ac6558-4baf-4db0-b8d6-4a8dd117ea6f","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-11-03T15:11:18.813Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"42ac6558-4baf-4db0-b8d6-4a8dd117ea6f","creating worker for group":"small-group","index":0,"total":1,"replicaIndex":0}
{"level":"info","ts":"2026-11-03T15:11:18.813Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"42ac6558-4baf-4db0-b8d6-4a8dd117ea6f","nodeType":"worker","rayStartParams":{"address":"long-running-2zj8b-head-svc.test-ns-5lqjm.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-11-03T15:11:18.813Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"42ac6558-4baf-4db0-b8d6-4a8dd117ea6f","rayStartCmd":"ray start  --address=long-running-2zj8b-head-svc.test-ns-5lqjm.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-11-03T15:11:18.813Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"42ac6558-4baf-4db0-b8d6-4a8dd117ea6f","rayNodeType":"worker","generatedCmd":"ulimit -n ${RAY_START_ULIMIT_OPEN_FILES:-65536}; ray start  --address=long-running-2zj8b-head-svc.test-ns-5lqjm.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-11-03T15:11:18.814Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"42ac6558-4baf-4db0-b8d6-4a8dd117ea6f","enabled":true}
{"level":"info","ts":"2026-11-03T15:11:18.874Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"42ac6558-4baf-4db0-b8d6-4a8dd117ea6f","name":"long-running-2zj8b-small-group-worker-mqllk"}
{"level":"info","ts":"2026-11-03T15:11:18.875Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"42ac6558-4baf-4db0-b8d6-4a8dd117ea6f","podName":"long-running-2zj8b-head-cklc6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:11:18.875Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"42ac6558-4baf-4db0-b8d6-4a8dd117ea6f","name":"long-running-2zj8b","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:12Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:18Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:11:18.935Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"ed21b3d7-8de1-438f-b613-3c18987dec38"}
{"level":"info","ts":"2026-11-03T15:11:18.935Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"ed21b3d7-8de1-438f-b613-3c18987dec38"}
{"level":"info","ts":"2026-11-03T15:11:18.935Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"ed21b3d7-8de1-438f-b613-3c18987dec38","1 head service found":"long-running-2zj8b-head-svc"}
{"level":"info","ts":"2026-11-03T15:11:18.935Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"5bbf0916-fd46-4525-b82f-e5593037f917","cluster":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:11:18.935Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"ed21b3d7-8de1-438f-b613-3c18987dec38","Found 1 head Pod":"long-running-2zj8b-head-cklc6","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:11:18.935Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"ed21b3d7-8de1-438f-b613-3c18987dec38","head Pod":"long-running-2zj8b-head-cklc6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-2zj8b-head-cklc6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:11:18.935Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"ed21b3d7-8de1-438f-b613-3c18987dec38","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:11:18.935Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"ed21b3d7-8de1-438f-b613-3c18987dec38","worker Pod":"long-running-2zj8b-small-group-worker-mqllk","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-2zj8b-small-group-worker-mqllk. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:11:18.935Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"ed21b3d7-8de1-438f-b613-3c18987dec38","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:11:18.935Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"ed21b3d7-8de1-438f-b613-3c18987dec38","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:11:18.935Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"ed21b3d7-8de1-438f-b613-3c18987dec38","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:11:18.935Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"ed21b3d7-8de1-438f-b613-3c18987dec38","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:11:18.936Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"ed21b3d7-8de1-438f-b613-3c18987dec38","podName":"long-running-2zj8b-head-cklc6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:11:18.936Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"b78b0cd6-8289-4712-b36c-3b21ff6e6d80","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:11:18.936Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"b78b0cd6-8289-4712-b36c-3b21ff6e6d80","name":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:11:18.936Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"ed21b3d7-8de1-438f-b613-3c18987dec38","podName":"long-running-2zj8b-head-cklc6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:11:18.936Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"b78b0cd6-8289-4712-b36c-3b21ff6e6d80","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:11:18.936Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"ed21b3d7-8de1-438f-b613-3c18987dec38","name":"long-running-2zj8b","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:12Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:18Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:11:18.936Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"b78b0cd6-8289-4712-b36c-3b21ff6e6d80"}
{"level":"info","ts":"2026-11-03T15:11:18.936Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"b78b0cd6-8289-4712-b36c-3b21ff6e6d80","RayCluster":"long-running-2zj8b","State":""}
{"level":"info","ts":"2026-11-03T15:11:18.936Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"b78b0cd6-8289-4712-b36c-3b21ff6e6d80","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:12Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:18Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:11:18.936Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"b78b0cd6-8289-4712-b36c-3b21ff6e6d80","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:11:18.990Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"ed21b3d7-8de1-438f-b613-3c18987dec38","name":"long-running-2zj8b","error":"Operation cannot be fulfilled on rayclusters.ray.io \"long-running-2zj8b\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-5lqjm","name":"long-running-2zj8b"}}
{"level":"info","ts":"2026-11-03T15:11:18.990Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"ed21b3d7-8de1-438f-b613-3c18987dec38"}
{"level":"error","ts":"2026-11-03T15:11:18.990Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"ed21b3d7-8de1-438f-b613-3c18987dec38","error":"Operation cannot be fulfilled on rayclusters.ray.io \"long-running-2zj8b\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:11:18.991Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"724cf65c-66d1-4d8e-90c8-f64f5e95111b"}
{"level":"info","ts":"2026-11-03T15:11:18.991Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"724cf65c-66d1-4d8e-90c8-f64f5e95111b"}
{"level":"info","ts":"2026-11-03T15:11:18.991Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"724cf65c-66d1-4d8e-90c8-f64f5e95111b","1 head service found":"long-running-2zj8b-head-svc"}
{"level":"info","ts":"2026-11-03T15:11:18.991Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"724cf65c-66d1-4d8e-90c8-f64f5e95111b","Found 1 head Pod":"long-running-2zj8b-head-cklc6","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:11:18.991Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"724cf65c-66d1-4d8e-90c8-f64f5e95111b","head Pod":"long-running-2zj8b-head-cklc6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-2zj8b-head-cklc6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:11:18.991Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"724cf65c-66d1-4d8e-90c8-f64f5e95111b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:11:18.991Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"724cf65c-66d1-4d8e-90c8-f64f5e95111b","worker Pod":"long-running-2zj8b-small-group-worker-mqllk","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-2zj8b-small-group-worker-mqllk. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:11:18.991Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"724cf65c-66d1-4d8e-90c8-f64f5e95111b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:11:18.991Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"724cf65c-66d1-4d8e-90c8-f64f5e95111b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:11:18.991Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"724cf65c-66d1-4d8e-90c8-f64f5e95111b","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:11:18.991Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"724cf65c-66d1-4d8e-90c8-f64f5e95111b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:11:18.991Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"724cf65c-66d1-4d8e-90c8-f64f5e95111b","podName":"long-running-2zj8b-head-cklc6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:11:18.991Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"724cf65c-66d1-4d8e-90c8-f64f5e95111b","podName":"long-running-2zj8b-head-cklc6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:11:18.991Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"724cf65c-66d1-4d8e-90c8-f64f5e95111b","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:11:18.991Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"724cf65c-66d1-4d8e-90c8-f64f5e95111b","seconds":300}
{"level":"info","ts":"2026-11-03T15:11:18.997Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"a92ceea4-bead-4e7e-aa85-3390b34796d7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:11:18.997Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"a92ceea4-bead-4e7e-aa85-3390b34796d7","name":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:11:18.997Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"a92ceea4-bead-4e7e-aa85-3390b34796d7","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:11:18.997Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"a92ceea4-bead-4e7e-aa85-3390b34796d7"}
{"level":"info","ts":"2026-11-03T15:11:18.997Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"a92ceea4-bead-4e7e-aa85-3390b34796d7","RayCluster":"long-running-2zj8b","State":""}
{"level":"info","ts":"2026-11-03T15:11:18.997Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"a92ceea4-bead-4e7e-aa85-3390b34796d7","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:18Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:18Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:11:18.999Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"91e24dae-53b7-48a5-ab6c-506e76a3d328"}
{"level":"info","ts":"2026-11-03T15:11:18.999Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"91e24dae-53b7-48a5-ab6c-506e76a3d328"}
{"level":"info","ts":"2026-11-03T15:11:18.999Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"91e24dae-53b7-48a5-ab6c-506e76a3d328","1 head service found":"long-running-2zj8b-head-svc"}
{"level":"info","ts":"2026-11-03T15:11:18.999Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"91e24dae-53b7-48a5-ab6c-506e76a3d328","Found 1 head Pod":"long-running-2zj8b-head-cklc6","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:11:18.999Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"91e24dae-53b7-48a5-ab6c-506e76a3d328","head Pod":"long-running-2zj8b-head-cklc6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-2zj8b-head-cklc6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:11:18.999Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"91e24dae-53b7-48a5-ab6c-506e76a3d328","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:11:18.999Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"91e24dae-53b7-48a5-ab6c-506e76a3d328","worker Pod":"long-running-2zj8b-small-group-worker-mqllk","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-2zj8b-small-group-worker-mqllk. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:11:18.999Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"91e24dae-53b7-48a5-ab6c-506e76a3d328","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:11:18.999Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"91e24dae-53b7-48a5-ab6c-506e76a3d328","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:11:18.999Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"91e24dae-53b7-48a5-ab6c-506e76a3d328","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:11:18.999Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"91e24dae-53b7-48a5-ab6c-506e76a3d328","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:11:18.999Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"91e24dae-53b7-48a5-ab6c-506e76a3d328","podName":"long-running-2zj8b-head-cklc6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:11:18.999Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"91e24dae-53b7-48a5-ab6c-506e76a3d328","podName":"long-running-2zj8b-head-cklc6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:11:18.999Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"91e24dae-53b7-48a5-ab6c-506e76a3d328","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:11:18.999Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"91e24dae-53b7-48a5-ab6c-506e76a3d328","seconds":300}
{"level":"info","ts":"2026-11-03T15:11:19.409Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"67b6e9ea-c1c2-4974-9956-e832aa8ebd2d"}
{"level":"info","ts":"2026-11-03T15:11:19.409Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"67b6e9ea-c1c2-4974-9956-e832aa8ebd2d"}
{"level":"info","ts":"2026-11-03T15:11:19.409Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"67b6e9ea-c1c2-4974-9956-e832aa8ebd2d","1 head service found":"long-running-2zj8b-head-svc"}
{"level":"info","ts":"2026-11-03T15:11:19.409Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"67b6e9ea-c1c2-4974-9956-e832aa8ebd2d","Found 1 head Pod":"long-running-2zj8b-head-cklc6","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:11:19.409Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"67b6e9ea-c1c2-4974-9956-e832aa8ebd2d","head Pod":"long-running-2zj8b-head-cklc6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-2zj8b-head-cklc6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:11:19.409Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"67b6e9ea-c1c2-4974-9956-e832aa8ebd2d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:11:19.409Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"67b6e9ea-c1c2-4974-9956-e832aa8ebd2d","worker Pod":"long-running-2zj8b-small-group-worker-mqllk","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-2zj8b-small-group-worker-mqllk. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:11:19.409Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"67b6e9ea-c1c2-4974-9956-e832aa8ebd2d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:11:19.409Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"67b6e9ea-c1c2-4974-9956-e832aa8ebd2d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:11:19.409Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"67b6e9ea-c1c2-4974-9956-e832aa8ebd2d","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:11:19.409Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"67b6e9ea-c1c2-4974-9956-e832aa8ebd2d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:11:19.409Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"67b6e9ea-c1c2-4974-9956-e832aa8ebd2d","podName":"long-running-2zj8b-head-cklc6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:11:19.409Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"67b6e9ea-c1c2-4974-9956-e832aa8ebd2d","podName":"long-running-2zj8b-head-cklc6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:11:19.409Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"67b6e9ea-c1c2-4974-9956-e832aa8ebd2d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:11:19.409Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"67b6e9ea-c1c2-4974-9956-e832aa8ebd2d","seconds":300}
{"level":"info","ts":"2026-11-03T15:11:19.625Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"830ffe05-fd9b-4d9b-827a-444cd03b3d53","DeletionTimestamp":"2026-11-03 15:11:19 +0000 UTC"}
{"level":"info","ts":"2026-11-03T15:11:19.625Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"830ffe05-fd9b-4d9b-827a-444cd03b3d53","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-11-03T15:11:19.675Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"0eb83de1-e11b-4a43-bd62-fd1265b881af"}
{"level":"info","ts":"2026-11-03T15:11:19.675Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"0eb83de1-e11b-4a43-bd62-fd1265b881af"}
{"level":"info","ts":"2026-11-03T15:11:19.675Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"0eb83de1-e11b-4a43-bd62-fd1265b881af","1 head service found":"long-running-2zj8b-head-svc"}
{"level":"info","ts":"2026-11-03T15:11:19.675Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"0eb83de1-e11b-4a43-bd62-fd1265b881af","Found 1 head Pod":"long-running-2zj8b-head-cklc6","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:11:19.675Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"0eb83de1-e11b-4a43-bd62-fd1265b881af","head Pod":"long-running-2zj8b-head-cklc6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-2zj8b-head-cklc6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:11:19.675Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"0eb83de1-e11b-4a43-bd62-fd1265b881af","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:11:19.675Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"0eb83de1-e11b-4a43-bd62-fd1265b881af","worker Pod":"long-running-2zj8b-small-group-worker-mqllk","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-2zj8b-small-group-worker-mqllk. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:11:19.675Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"0eb83de1-e11b-4a43-bd62-fd1265b881af","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:11:19.675Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"0eb83de1-e11b-4a43-bd62-fd1265b881af","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:11:19.675Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"0eb83de1-e11b-4a43-bd62-fd1265b881af","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:11:19.675Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"0eb83de1-e11b-4a43-bd62-fd1265b881af","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:11:19.676Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"0eb83de1-e11b-4a43-bd62-fd1265b881af","podName":"long-running-2zj8b-head-cklc6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:11:19.676Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"0eb83de1-e11b-4a43-bd62-fd1265b881af","podName":"long-running-2zj8b-head-cklc6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:11:19.676Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"0eb83de1-e11b-4a43-bd62-fd1265b881af","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:11:19.676Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"0eb83de1-e11b-4a43-bd62-fd1265b881af","seconds":300}
{"level":"info","ts":"2026-11-03T15:11:19.687Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"98edf487-215b-48ee-930c-114e06c194d6","DeletionTimestamp":"2026-11-03 15:11:19 +0000 UTC"}
{"level":"info","ts":"2026-11-03T15:11:19.687Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"98edf487-215b-48ee-930c-114e06c194d6","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-11-03T15:11:19.750Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"counter","namespace":"test-ns-mvz6h"},"reconcileID":"438bf3f5-9cc4-47b1-a3c6-c58d96e3e733"}
{"level":"info","ts":"2026-11-03T15:11:19.750Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"counter-auth","namespace":"test-ns-mvz6h"},"reconcileID":"95a15e47-528a-4ead-94aa-29589f8c8df9"}
{"level":"info","ts":"2026-11-03T15:11:19.919Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"cb0d0d72-db44-4403-b7cb-4a33ac4ee34d"}
{"level":"info","ts":"2026-11-03T15:11:19.919Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"cb0d0d72-db44-4403-b7cb-4a33ac4ee34d"}
{"level":"info","ts":"2026-11-03T15:11:19.919Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"cb0d0d72-db44-4403-b7cb-4a33ac4ee34d","1 head service found":"long-running-2zj8b-head-svc"}
{"level":"info","ts":"2026-11-03T15:11:19.919Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"cb0d0d72-db44-4403-b7cb-4a33ac4ee34d","Found 1 head Pod":"long-running-2zj8b-head-cklc6","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:11:19.919Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"cb0d0d72-db44-4403-b7cb-4a33ac4ee34d","head Pod":"long-running-2zj8b-head-cklc6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-2zj8b-head-cklc6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:11:19.919Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"cb0d0d72-db44-4403-b7cb-4a33ac4ee34d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:11:19.919Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"cb0d0d72-db44-4403-b7cb-4a33ac4ee34d","worker Pod":"long-running-2zj8b-small-group-worker-mqllk","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-2zj8b-small-group-worker-mqllk. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:11:19.919Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"cb0d0d72-db44-4403-b7cb-4a33ac4ee34d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:11:19.919Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"cb0d0d72-db44-4403-b7cb-4a33ac4ee34d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:11:19.919Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"cb0d0d72-db44-4403-b7cb-4a33ac4ee34d","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:11:19.919Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"cb0d0d72-db44-4403-b7cb-4a33ac4ee34d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:11:19.919Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"cb0d0d72-db44-4403-b7cb-4a33ac4ee34d","podName":"long-running-2zj8b-head-cklc6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:11:19.919Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"cb0d0d72-db44-4403-b7cb-4a33ac4ee34d","podName":"long-running-2zj8b-head-cklc6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:11:19.919Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"cb0d0d72-db44-4403-b7cb-4a33ac4ee34d","name":"long-running-2zj8b","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:18Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:11:19.980Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"8f7ec5b1-f366-4ce4-932b-c6d809d28321","cluster":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:11:19.980Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"b560506b-f539-4707-a2dd-7a9924a98c11","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:11:19.980Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"b560506b-f539-4707-a2dd-7a9924a98c11","name":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:11:19.980Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"b560506b-f539-4707-a2dd-7a9924a98c11","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:11:19.981Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"b560506b-f539-4707-a2dd-7a9924a98c11"}
{"level":"info","ts":"2026-11-03T15:11:19.981Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"b560506b-f539-4707-a2dd-7a9924a98c11","RayCluster":"long-running-2zj8b","State":""}
{"level":"info","ts":"2026-11-03T15:11:19.981Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"b560506b-f539-4707-a2dd-7a9924a98c11","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:18Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:11:19.981Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"b560506b-f539-4707-a2dd-7a9924a98c11","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:11:20.042Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"78c4089f-eac6-469e-a162-664b4942e216","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:11:20.042Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"78c4089f-eac6-469e-a162-664b4942e216","name":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:11:20.042Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"78c4089f-eac6-469e-a162-664b4942e216","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:11:20.042Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"78c4089f-eac6-469e-a162-664b4942e216"}
{"level":"info","ts":"2026-11-03T15:11:20.042Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"78c4089f-eac6-469e-a162-664b4942e216","RayCluster":"long-running-2zj8b","State":""}
{"level":"info","ts":"2026-11-03T15:11:20.042Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"78c4089f-eac6-469e-a162-664b4942e216","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:11:20.283Z","logger":"controllers.RayClusterMTLS","msg":"Updating head certificate","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"8f7ec5b1-f366-4ce4-932b-c6d809d28321","certificate":"ray-head-cert-long-running-2zj8b","ipAddresses":["10.132.0.35","127.0.0.1"]}
{"level":"info","ts":"2026-11-03T15:11:20.691Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"8f7ec5b1-f366-4ce4-932b-c6d809d28321","certificate":"ray-head-cert-long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:11:20.691Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"8f7ec5b1-f366-4ce4-932b-c6d809d28321"}
{"level":"info","ts":"2026-11-03T15:11:20.780Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"e8ee39ae-1a96-4fbc-906f-ceb5caa5a465"}
{"level":"info","ts":"2026-11-03T15:11:20.780Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"e8ee39ae-1a96-4fbc-906f-ceb5caa5a465"}
{"level":"info","ts":"2026-11-03T15:11:20.780Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"e8ee39ae-1a96-4fbc-906f-ceb5caa5a465","1 head service found":"long-running-2zj8b-head-svc"}
{"level":"info","ts":"2026-11-03T15:11:20.781Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"e8ee39ae-1a96-4fbc-906f-ceb5caa5a465","Found 1 head Pod":"long-running-2zj8b-head-cklc6","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:11:20.781Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"e8ee39ae-1a96-4fbc-906f-ceb5caa5a465","head Pod":"long-running-2zj8b-head-cklc6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-2zj8b-head-cklc6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:11:20.781Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"e8ee39ae-1a96-4fbc-906f-ceb5caa5a465","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:11:20.781Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"e8ee39ae-1a96-4fbc-906f-ceb5caa5a465","worker Pod":"long-running-2zj8b-small-group-worker-mqllk","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-2zj8b-small-group-worker-mqllk. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:11:20.781Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"e8ee39ae-1a96-4fbc-906f-ceb5caa5a465","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:11:20.781Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"e8ee39ae-1a96-4fbc-906f-ceb5caa5a465","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:11:20.781Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"e8ee39ae-1a96-4fbc-906f-ceb5caa5a465","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:11:20.781Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"e8ee39ae-1a96-4fbc-906f-ceb5caa5a465","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:11:20.781Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"e8ee39ae-1a96-4fbc-906f-ceb5caa5a465","podName":"long-running-2zj8b-head-cklc6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:11:20.781Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"e8ee39ae-1a96-4fbc-906f-ceb5caa5a465","podName":"long-running-2zj8b-head-cklc6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:11:20.781Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"e8ee39ae-1a96-4fbc-906f-ceb5caa5a465","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:11:20.781Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"e8ee39ae-1a96-4fbc-906f-ceb5caa5a465","seconds":300}
{"level":"info","ts":"2026-11-03T15:11:23.042Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"57ba31bc-c899-4268-8970-724b570d8106","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:11:23.042Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"57ba31bc-c899-4268-8970-724b570d8106","name":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:11:23.042Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"57ba31bc-c899-4268-8970-724b570d8106","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:11:23.043Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"57ba31bc-c899-4268-8970-724b570d8106"}
{"level":"info","ts":"2026-11-03T15:11:23.043Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"57ba31bc-c899-4268-8970-724b570d8106","RayCluster":"long-running-2zj8b","State":""}
{"level":"info","ts":"2026-11-03T15:11:23.043Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"57ba31bc-c899-4268-8970-724b570d8106","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:11:26.043Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"d653d9ec-2729-4406-af7e-5d6e17b0622e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:11:26.043Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"d653d9ec-2729-4406-af7e-5d6e17b0622e","name":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:11:26.043Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"d653d9ec-2729-4406-af7e-5d6e17b0622e","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:11:26.044Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"d653d9ec-2729-4406-af7e-5d6e17b0622e"}
{"level":"info","ts":"2026-11-03T15:11:26.044Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"d653d9ec-2729-4406-af7e-5d6e17b0622e","RayCluster":"long-running-2zj8b","State":""}
{"level":"info","ts":"2026-11-03T15:11:26.044Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"d653d9ec-2729-4406-af7e-5d6e17b0622e","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:11:29.045Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"80a9c148-319e-4660-9d20-d10fea49c6b6","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:11:29.045Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"80a9c148-319e-4660-9d20-d10fea49c6b6","name":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:11:29.045Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"80a9c148-319e-4660-9d20-d10fea49c6b6","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:11:29.045Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"80a9c148-319e-4660-9d20-d10fea49c6b6"}
{"level":"info","ts":"2026-11-03T15:11:29.045Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"80a9c148-319e-4660-9d20-d10fea49c6b6","RayCluster":"long-running-2zj8b","State":""}
{"level":"info","ts":"2026-11-03T15:11:29.045Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"80a9c148-319e-4660-9d20-d10fea49c6b6","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:11:32.046Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"491ba9bd-e024-40fd-9a70-f8dbaef03248","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:11:32.046Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"491ba9bd-e024-40fd-9a70-f8dbaef03248","name":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:11:32.046Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"491ba9bd-e024-40fd-9a70-f8dbaef03248","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:11:32.047Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"491ba9bd-e024-40fd-9a70-f8dbaef03248"}
{"level":"info","ts":"2026-11-03T15:11:32.047Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"491ba9bd-e024-40fd-9a70-f8dbaef03248","RayCluster":"long-running-2zj8b","State":""}
{"level":"info","ts":"2026-11-03T15:11:32.047Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"491ba9bd-e024-40fd-9a70-f8dbaef03248","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:11:35.048Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"ae43ab8c-3e05-45cd-8ee5-87b2c743b55d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:11:35.048Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"ae43ab8c-3e05-45cd-8ee5-87b2c743b55d","name":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:11:35.048Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"ae43ab8c-3e05-45cd-8ee5-87b2c743b55d","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:11:35.048Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"ae43ab8c-3e05-45cd-8ee5-87b2c743b55d"}
{"level":"info","ts":"2026-11-03T15:11:35.048Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"ae43ab8c-3e05-45cd-8ee5-87b2c743b55d","RayCluster":"long-running-2zj8b","State":""}
{"level":"info","ts":"2026-11-03T15:11:35.048Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"ae43ab8c-3e05-45cd-8ee5-87b2c743b55d","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:11:38.049Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"f58499c3-e705-4bc9-bb67-7abea415fd9e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:11:38.049Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"f58499c3-e705-4bc9-bb67-7abea415fd9e","name":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:11:38.049Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"f58499c3-e705-4bc9-bb67-7abea415fd9e","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:11:38.050Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"f58499c3-e705-4bc9-bb67-7abea415fd9e"}
{"level":"info","ts":"2026-11-03T15:11:38.050Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"f58499c3-e705-4bc9-bb67-7abea415fd9e","RayCluster":"long-running-2zj8b","State":""}
{"level":"info","ts":"2026-11-03T15:11:38.050Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"f58499c3-e705-4bc9-bb67-7abea415fd9e","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:11:41.051Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"406b9393-3e3d-485c-b91d-bd195344fed7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:11:41.051Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"406b9393-3e3d-485c-b91d-bd195344fed7","name":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:11:41.051Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"406b9393-3e3d-485c-b91d-bd195344fed7","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:11:41.051Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"406b9393-3e3d-485c-b91d-bd195344fed7"}
{"level":"info","ts":"2026-11-03T15:11:41.051Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"406b9393-3e3d-485c-b91d-bd195344fed7","RayCluster":"long-running-2zj8b","State":""}
{"level":"info","ts":"2026-11-03T15:11:41.051Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"406b9393-3e3d-485c-b91d-bd195344fed7","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:11:44.051Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"edc5028c-46ed-4512-859f-6cc9f8f5d543","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:11:44.051Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"edc5028c-46ed-4512-859f-6cc9f8f5d543","name":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:11:44.051Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"edc5028c-46ed-4512-859f-6cc9f8f5d543","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:11:44.051Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"edc5028c-46ed-4512-859f-6cc9f8f5d543"}
{"level":"info","ts":"2026-11-03T15:11:44.052Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"edc5028c-46ed-4512-859f-6cc9f8f5d543","RayCluster":"long-running-2zj8b","State":""}
{"level":"info","ts":"2026-11-03T15:11:44.052Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"edc5028c-46ed-4512-859f-6cc9f8f5d543","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:11:47.052Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"dfb95a4a-db40-4781-9bac-0a8223a5119e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:11:47.052Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"dfb95a4a-db40-4781-9bac-0a8223a5119e","name":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:11:47.052Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"dfb95a4a-db40-4781-9bac-0a8223a5119e","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:11:47.053Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"dfb95a4a-db40-4781-9bac-0a8223a5119e"}
{"level":"info","ts":"2026-11-03T15:11:47.053Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"dfb95a4a-db40-4781-9bac-0a8223a5119e","RayCluster":"long-running-2zj8b","State":""}
{"level":"info","ts":"2026-11-03T15:11:47.053Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"dfb95a4a-db40-4781-9bac-0a8223a5119e","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:11:50.053Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"ce7fa2dd-4a9f-41a9-b101-cb075b405120","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:11:50.053Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"ce7fa2dd-4a9f-41a9-b101-cb075b405120","name":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:11:50.053Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"ce7fa2dd-4a9f-41a9-b101-cb075b405120","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:11:50.054Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"ce7fa2dd-4a9f-41a9-b101-cb075b405120"}
{"level":"info","ts":"2026-11-03T15:11:50.054Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"ce7fa2dd-4a9f-41a9-b101-cb075b405120","RayCluster":"long-running-2zj8b","State":""}
{"level":"info","ts":"2026-11-03T15:11:50.054Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"ce7fa2dd-4a9f-41a9-b101-cb075b405120","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:11:50.692Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"1ed79668-820b-4912-9b8e-23e0ff249556","cluster":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:11:51.148Z","logger":"controllers.RayClusterMTLS","msg":"Updating worker certificate","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"1ed79668-820b-4912-9b8e-23e0ff249556","certificate":"ray-worker-cert-long-running-2zj8b","ipAddresses":["10.134.0.31","127.0.0.1"]}
{"level":"info","ts":"2026-11-03T15:11:51.462Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"1ed79668-820b-4912-9b8e-23e0ff249556","certificate":"ray-worker-cert-long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:11:51.462Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"1ed79668-820b-4912-9b8e-23e0ff249556"}
{"level":"info","ts":"2026-11-03T15:11:53.054Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"ac2da931-7fc1-41ce-b3d1-5d6518965c11","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:11:53.054Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"ac2da931-7fc1-41ce-b3d1-5d6518965c11","name":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:11:53.054Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"ac2da931-7fc1-41ce-b3d1-5d6518965c11","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:11:53.054Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"ac2da931-7fc1-41ce-b3d1-5d6518965c11"}
{"level":"info","ts":"2026-11-03T15:11:53.054Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"ac2da931-7fc1-41ce-b3d1-5d6518965c11","RayCluster":"long-running-2zj8b","State":""}
{"level":"info","ts":"2026-11-03T15:11:53.054Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"ac2da931-7fc1-41ce-b3d1-5d6518965c11","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:11:56.056Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"4c14164b-eda3-40bd-8019-016faa5368ba","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:11:56.056Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"4c14164b-eda3-40bd-8019-016faa5368ba","name":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:11:56.056Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"4c14164b-eda3-40bd-8019-016faa5368ba","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:11:56.056Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"4c14164b-eda3-40bd-8019-016faa5368ba"}
{"level":"info","ts":"2026-11-03T15:11:56.056Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"4c14164b-eda3-40bd-8019-016faa5368ba","RayCluster":"long-running-2zj8b","State":""}
{"level":"info","ts":"2026-11-03T15:11:56.056Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"4c14164b-eda3-40bd-8019-016faa5368ba","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:11:59.057Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"872a38e1-0e82-494d-ba79-8cfe999dc2f2","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:11:59.057Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"872a38e1-0e82-494d-ba79-8cfe999dc2f2","name":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:11:59.057Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"872a38e1-0e82-494d-ba79-8cfe999dc2f2","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:11:59.057Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"872a38e1-0e82-494d-ba79-8cfe999dc2f2"}
{"level":"info","ts":"2026-11-03T15:11:59.057Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"872a38e1-0e82-494d-ba79-8cfe999dc2f2","RayCluster":"long-running-2zj8b","State":""}
{"level":"info","ts":"2026-11-03T15:11:59.057Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"872a38e1-0e82-494d-ba79-8cfe999dc2f2","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:12:02.058Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"d5c614e0-2ce2-48be-8821-1a7057c0acfb","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:12:02.058Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"d5c614e0-2ce2-48be-8821-1a7057c0acfb","name":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:12:02.058Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"d5c614e0-2ce2-48be-8821-1a7057c0acfb","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:12:02.058Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"d5c614e0-2ce2-48be-8821-1a7057c0acfb"}
{"level":"info","ts":"2026-11-03T15:12:02.059Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"d5c614e0-2ce2-48be-8821-1a7057c0acfb","RayCluster":"long-running-2zj8b","State":""}
{"level":"info","ts":"2026-11-03T15:12:02.059Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"d5c614e0-2ce2-48be-8821-1a7057c0acfb","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:12:05.060Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"a034b61a-c5c8-4d6c-8a7c-7ef0eb3c9f53","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:12:05.060Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"a034b61a-c5c8-4d6c-8a7c-7ef0eb3c9f53","name":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:12:05.060Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"a034b61a-c5c8-4d6c-8a7c-7ef0eb3c9f53","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:12:05.060Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"a034b61a-c5c8-4d6c-8a7c-7ef0eb3c9f53"}
{"level":"info","ts":"2026-11-03T15:12:05.060Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"a034b61a-c5c8-4d6c-8a7c-7ef0eb3c9f53","RayCluster":"long-running-2zj8b","State":""}
{"level":"info","ts":"2026-11-03T15:12:05.060Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"a034b61a-c5c8-4d6c-8a7c-7ef0eb3c9f53","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:12:08.060Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"b3b5e359-66f2-4953-9b6d-2b5c96e39340","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:12:08.060Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"b3b5e359-66f2-4953-9b6d-2b5c96e39340","name":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:12:08.060Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"b3b5e359-66f2-4953-9b6d-2b5c96e39340","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:12:08.061Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"b3b5e359-66f2-4953-9b6d-2b5c96e39340"}
{"level":"info","ts":"2026-11-03T15:12:08.061Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"b3b5e359-66f2-4953-9b6d-2b5c96e39340","RayCluster":"long-running-2zj8b","State":""}
{"level":"info","ts":"2026-11-03T15:12:08.061Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"b3b5e359-66f2-4953-9b6d-2b5c96e39340","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:12:11.062Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"97f17718-7106-4833-844f-cab9d3ad5880","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:12:11.062Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"97f17718-7106-4833-844f-cab9d3ad5880","name":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:12:11.062Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"97f17718-7106-4833-844f-cab9d3ad5880","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:12:11.062Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"97f17718-7106-4833-844f-cab9d3ad5880"}
{"level":"info","ts":"2026-11-03T15:12:11.062Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"97f17718-7106-4833-844f-cab9d3ad5880","RayCluster":"long-running-2zj8b","State":""}
{"level":"info","ts":"2026-11-03T15:12:11.062Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"97f17718-7106-4833-844f-cab9d3ad5880","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:12:14.063Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"f70389ba-e1b9-4431-a255-02a621803e2f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:12:14.063Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"f70389ba-e1b9-4431-a255-02a621803e2f","name":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:12:14.063Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"f70389ba-e1b9-4431-a255-02a621803e2f","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:12:14.064Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"f70389ba-e1b9-4431-a255-02a621803e2f"}
{"level":"info","ts":"2026-11-03T15:12:14.064Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"f70389ba-e1b9-4431-a255-02a621803e2f","RayCluster":"long-running-2zj8b","State":""}
{"level":"info","ts":"2026-11-03T15:12:14.064Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"f70389ba-e1b9-4431-a255-02a621803e2f","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:12:17.065Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"75d5587f-bd85-40be-93f6-c90e7bedec6f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:12:17.065Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"75d5587f-bd85-40be-93f6-c90e7bedec6f","name":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:12:17.065Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"75d5587f-bd85-40be-93f6-c90e7bedec6f","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:12:17.065Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"75d5587f-bd85-40be-93f6-c90e7bedec6f"}
{"level":"info","ts":"2026-11-03T15:12:17.065Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"75d5587f-bd85-40be-93f6-c90e7bedec6f","RayCluster":"long-running-2zj8b","State":""}
{"level":"info","ts":"2026-11-03T15:12:17.065Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"75d5587f-bd85-40be-93f6-c90e7bedec6f","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:12:20.066Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"609b72a5-fb66-4b87-ad7b-3844a19b2da0","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:12:20.066Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"609b72a5-fb66-4b87-ad7b-3844a19b2da0","name":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:12:20.066Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"609b72a5-fb66-4b87-ad7b-3844a19b2da0","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:12:20.066Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"609b72a5-fb66-4b87-ad7b-3844a19b2da0"}
{"level":"info","ts":"2026-11-03T15:12:20.066Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"609b72a5-fb66-4b87-ad7b-3844a19b2da0","RayCluster":"long-running-2zj8b","State":""}
{"level":"info","ts":"2026-11-03T15:12:20.066Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"609b72a5-fb66-4b87-ad7b-3844a19b2da0","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:12:21.462Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"cf450f12-c980-4ff3-8299-7ecca142eeac","cluster":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:12:23.067Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"541b3af0-5b37-45c6-a844-5d7d428238d5","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:12:23.067Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"541b3af0-5b37-45c6-a844-5d7d428238d5","name":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:12:23.067Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"541b3af0-5b37-45c6-a844-5d7d428238d5","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:12:23.067Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"541b3af0-5b37-45c6-a844-5d7d428238d5"}
{"level":"info","ts":"2026-11-03T15:12:23.067Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"541b3af0-5b37-45c6-a844-5d7d428238d5","RayCluster":"long-running-2zj8b","State":""}
{"level":"info","ts":"2026-11-03T15:12:23.067Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"541b3af0-5b37-45c6-a844-5d7d428238d5","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:12:26.068Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"915c9f7d-e509-475a-abb2-53e373b3b5f0","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:12:26.068Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"915c9f7d-e509-475a-abb2-53e373b3b5f0","name":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:12:26.068Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"915c9f7d-e509-475a-abb2-53e373b3b5f0","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:12:26.068Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"915c9f7d-e509-475a-abb2-53e373b3b5f0"}
{"level":"info","ts":"2026-11-03T15:12:26.068Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"915c9f7d-e509-475a-abb2-53e373b3b5f0","RayCluster":"long-running-2zj8b","State":""}
{"level":"info","ts":"2026-11-03T15:12:26.068Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"915c9f7d-e509-475a-abb2-53e373b3b5f0","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:12:29.069Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"bcb97e12-639d-480e-b690-ebc0147a29a7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:12:29.069Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"bcb97e12-639d-480e-b690-ebc0147a29a7","name":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:12:29.069Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"bcb97e12-639d-480e-b690-ebc0147a29a7","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:12:29.069Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"bcb97e12-639d-480e-b690-ebc0147a29a7"}
{"level":"info","ts":"2026-11-03T15:12:29.069Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"bcb97e12-639d-480e-b690-ebc0147a29a7","RayCluster":"long-running-2zj8b","State":""}
{"level":"info","ts":"2026-11-03T15:12:29.069Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"bcb97e12-639d-480e-b690-ebc0147a29a7","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:12:32.071Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"e05ecade-92fa-481b-9caa-6dd2ca3dac12","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:12:32.071Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"e05ecade-92fa-481b-9caa-6dd2ca3dac12","name":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:12:32.071Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"e05ecade-92fa-481b-9caa-6dd2ca3dac12","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:12:32.071Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"e05ecade-92fa-481b-9caa-6dd2ca3dac12"}
{"level":"info","ts":"2026-11-03T15:12:32.071Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"e05ecade-92fa-481b-9caa-6dd2ca3dac12","RayCluster":"long-running-2zj8b","State":""}
{"level":"info","ts":"2026-11-03T15:12:32.071Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"e05ecade-92fa-481b-9caa-6dd2ca3dac12","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:12:35.071Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"105a8069-dd1a-41f9-924e-95aa957093d2","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:12:35.071Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"105a8069-dd1a-41f9-924e-95aa957093d2","name":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:12:35.071Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"105a8069-dd1a-41f9-924e-95aa957093d2","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:12:35.072Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"105a8069-dd1a-41f9-924e-95aa957093d2"}
{"level":"info","ts":"2026-11-03T15:12:35.072Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"105a8069-dd1a-41f9-924e-95aa957093d2","RayCluster":"long-running-2zj8b","State":""}
{"level":"info","ts":"2026-11-03T15:12:35.072Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"105a8069-dd1a-41f9-924e-95aa957093d2","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:12:38.073Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"dda861f3-742b-4215-97af-56b5fa5a0471","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:12:38.073Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"dda861f3-742b-4215-97af-56b5fa5a0471","name":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:12:38.073Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"dda861f3-742b-4215-97af-56b5fa5a0471","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:12:38.073Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"dda861f3-742b-4215-97af-56b5fa5a0471"}
{"level":"info","ts":"2026-11-03T15:12:38.073Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"dda861f3-742b-4215-97af-56b5fa5a0471","RayCluster":"long-running-2zj8b","State":""}
{"level":"info","ts":"2026-11-03T15:12:38.073Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"dda861f3-742b-4215-97af-56b5fa5a0471","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:12:41.074Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"067cba99-a011-4134-977f-442a9f2c5b93","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:12:41.074Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"067cba99-a011-4134-977f-442a9f2c5b93","name":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:12:41.074Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"067cba99-a011-4134-977f-442a9f2c5b93","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:12:41.074Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"067cba99-a011-4134-977f-442a9f2c5b93"}
{"level":"info","ts":"2026-11-03T15:12:41.074Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"067cba99-a011-4134-977f-442a9f2c5b93","RayCluster":"long-running-2zj8b","State":""}
{"level":"info","ts":"2026-11-03T15:12:41.074Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"067cba99-a011-4134-977f-442a9f2c5b93","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:12:41.133Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"75c6e50f-2883-414a-95d2-2f9e4a2ddcfe"}
{"level":"info","ts":"2026-11-03T15:12:41.133Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"75c6e50f-2883-414a-95d2-2f9e4a2ddcfe"}
{"level":"info","ts":"2026-11-03T15:12:41.133Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"75c6e50f-2883-414a-95d2-2f9e4a2ddcfe","1 head service found":"long-running-2zj8b-head-svc"}
{"level":"info","ts":"2026-11-03T15:12:41.134Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"75c6e50f-2883-414a-95d2-2f9e4a2ddcfe","Found 1 head Pod":"long-running-2zj8b-head-cklc6","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:12:41.134Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"75c6e50f-2883-414a-95d2-2f9e4a2ddcfe","head Pod":"long-running-2zj8b-head-cklc6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-2zj8b-head-cklc6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:12:41.134Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"75c6e50f-2883-414a-95d2-2f9e4a2ddcfe","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:12:41.134Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"75c6e50f-2883-414a-95d2-2f9e4a2ddcfe","worker Pod":"long-running-2zj8b-small-group-worker-mqllk","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-2zj8b-small-group-worker-mqllk. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:12:41.134Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"75c6e50f-2883-414a-95d2-2f9e4a2ddcfe","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:12:41.134Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"75c6e50f-2883-414a-95d2-2f9e4a2ddcfe","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:12:41.134Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"75c6e50f-2883-414a-95d2-2f9e4a2ddcfe","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:12:41.134Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"75c6e50f-2883-414a-95d2-2f9e4a2ddcfe","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:12:41.134Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"75c6e50f-2883-414a-95d2-2f9e4a2ddcfe","podName":"long-running-2zj8b-head-cklc6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:12:41.134Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"75c6e50f-2883-414a-95d2-2f9e4a2ddcfe","podName":"long-running-2zj8b-head-cklc6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:12:41.134Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"75c6e50f-2883-414a-95d2-2f9e4a2ddcfe","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:12:41.134Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"75c6e50f-2883-414a-95d2-2f9e4a2ddcfe","seconds":300}
{"level":"info","ts":"2026-11-03T15:12:42.142Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"7df4b8b0-2940-4802-bd08-642657e76d1e"}
{"level":"info","ts":"2026-11-03T15:12:42.142Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"7df4b8b0-2940-4802-bd08-642657e76d1e"}
{"level":"info","ts":"2026-11-03T15:12:42.142Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"7df4b8b0-2940-4802-bd08-642657e76d1e","1 head service found":"long-running-2zj8b-head-svc"}
{"level":"info","ts":"2026-11-03T15:12:42.142Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"7df4b8b0-2940-4802-bd08-642657e76d1e","Found 1 head Pod":"long-running-2zj8b-head-cklc6","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:12:42.142Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"7df4b8b0-2940-4802-bd08-642657e76d1e","head Pod":"long-running-2zj8b-head-cklc6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-2zj8b-head-cklc6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:12:42.142Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"7df4b8b0-2940-4802-bd08-642657e76d1e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:12:42.142Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"7df4b8b0-2940-4802-bd08-642657e76d1e","worker Pod":"long-running-2zj8b-small-group-worker-mqllk","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-2zj8b-small-group-worker-mqllk. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:12:42.142Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"7df4b8b0-2940-4802-bd08-642657e76d1e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:12:42.142Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"7df4b8b0-2940-4802-bd08-642657e76d1e","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:12:42.142Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"7df4b8b0-2940-4802-bd08-642657e76d1e","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:12:42.142Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"7df4b8b0-2940-4802-bd08-642657e76d1e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:12:42.142Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"7df4b8b0-2940-4802-bd08-642657e76d1e","podName":"long-running-2zj8b-head-cklc6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:12:42.142Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"7df4b8b0-2940-4802-bd08-642657e76d1e","podName":"long-running-2zj8b-head-cklc6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:12:42.143Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"7df4b8b0-2940-4802-bd08-642657e76d1e","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:12:42.143Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"7df4b8b0-2940-4802-bd08-642657e76d1e","seconds":300}
{"level":"info","ts":"2026-11-03T15:12:44.075Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"b3471f00-6d2a-4752-880c-80f67d2064f0","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:12:44.075Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"b3471f00-6d2a-4752-880c-80f67d2064f0","name":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:12:44.075Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"b3471f00-6d2a-4752-880c-80f67d2064f0","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:12:44.076Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"b3471f00-6d2a-4752-880c-80f67d2064f0"}
{"level":"info","ts":"2026-11-03T15:12:44.076Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"b3471f00-6d2a-4752-880c-80f67d2064f0","RayCluster":"long-running-2zj8b","State":""}
{"level":"info","ts":"2026-11-03T15:12:44.076Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"b3471f00-6d2a-4752-880c-80f67d2064f0","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:12:47.076Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"6edfdbbd-e8e9-45cc-8c10-af3cce894f19","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:12:47.076Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"6edfdbbd-e8e9-45cc-8c10-af3cce894f19","name":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:12:47.076Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"6edfdbbd-e8e9-45cc-8c10-af3cce894f19","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:12:47.076Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"6edfdbbd-e8e9-45cc-8c10-af3cce894f19"}
{"level":"info","ts":"2026-11-03T15:12:47.076Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"6edfdbbd-e8e9-45cc-8c10-af3cce894f19","RayCluster":"long-running-2zj8b","State":""}
{"level":"info","ts":"2026-11-03T15:12:47.076Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"6edfdbbd-e8e9-45cc-8c10-af3cce894f19","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:12:50.077Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"7a3cfe0d-1a0e-4597-8ac2-5a1196efc46e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:12:50.077Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"7a3cfe0d-1a0e-4597-8ac2-5a1196efc46e","name":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:12:50.077Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"7a3cfe0d-1a0e-4597-8ac2-5a1196efc46e","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:12:50.077Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"7a3cfe0d-1a0e-4597-8ac2-5a1196efc46e"}
{"level":"info","ts":"2026-11-03T15:12:50.077Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"7a3cfe0d-1a0e-4597-8ac2-5a1196efc46e","RayCluster":"long-running-2zj8b","State":""}
{"level":"info","ts":"2026-11-03T15:12:50.077Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"7a3cfe0d-1a0e-4597-8ac2-5a1196efc46e","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:12:53.078Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"384fd0f9-adda-4add-83e0-8a588d46f083","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:12:53.078Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"384fd0f9-adda-4add-83e0-8a588d46f083","name":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:12:53.078Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"384fd0f9-adda-4add-83e0-8a588d46f083","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:12:53.078Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"384fd0f9-adda-4add-83e0-8a588d46f083"}
{"level":"info","ts":"2026-11-03T15:12:53.078Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"384fd0f9-adda-4add-83e0-8a588d46f083","RayCluster":"long-running-2zj8b","State":""}
{"level":"info","ts":"2026-11-03T15:12:53.078Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"384fd0f9-adda-4add-83e0-8a588d46f083","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:12:56.079Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"2aaa67c1-a401-481b-92e7-62b020d9a5d7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:12:56.079Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"2aaa67c1-a401-481b-92e7-62b020d9a5d7","name":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:12:56.079Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"2aaa67c1-a401-481b-92e7-62b020d9a5d7","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:12:56.079Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"2aaa67c1-a401-481b-92e7-62b020d9a5d7"}
{"level":"info","ts":"2026-11-03T15:12:56.079Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"2aaa67c1-a401-481b-92e7-62b020d9a5d7","RayCluster":"long-running-2zj8b","State":""}
{"level":"info","ts":"2026-11-03T15:12:56.079Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"2aaa67c1-a401-481b-92e7-62b020d9a5d7","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:12:56.390Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"abc3d3fb-115e-4477-a6e1-bb9dd289ca1d"}
{"level":"info","ts":"2026-11-03T15:12:56.390Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"abc3d3fb-115e-4477-a6e1-bb9dd289ca1d"}
{"level":"info","ts":"2026-11-03T15:12:56.390Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"abc3d3fb-115e-4477-a6e1-bb9dd289ca1d","1 head service found":"long-running-2zj8b-head-svc"}
{"level":"info","ts":"2026-11-03T15:12:56.390Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"abc3d3fb-115e-4477-a6e1-bb9dd289ca1d","Found 1 head Pod":"long-running-2zj8b-head-cklc6","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:12:56.390Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"abc3d3fb-115e-4477-a6e1-bb9dd289ca1d","head Pod":"long-running-2zj8b-head-cklc6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-2zj8b-head-cklc6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:12:56.390Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"abc3d3fb-115e-4477-a6e1-bb9dd289ca1d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:12:56.390Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"abc3d3fb-115e-4477-a6e1-bb9dd289ca1d","worker Pod":"long-running-2zj8b-small-group-worker-mqllk","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-2zj8b-small-group-worker-mqllk. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:12:56.390Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"abc3d3fb-115e-4477-a6e1-bb9dd289ca1d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:12:56.390Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"abc3d3fb-115e-4477-a6e1-bb9dd289ca1d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:12:56.390Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"abc3d3fb-115e-4477-a6e1-bb9dd289ca1d","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:12:56.390Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"abc3d3fb-115e-4477-a6e1-bb9dd289ca1d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:12:56.390Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"abc3d3fb-115e-4477-a6e1-bb9dd289ca1d","podName":"long-running-2zj8b-head-cklc6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:12:56.390Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"abc3d3fb-115e-4477-a6e1-bb9dd289ca1d","podName":"long-running-2zj8b-head-cklc6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:12:56.390Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"abc3d3fb-115e-4477-a6e1-bb9dd289ca1d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:12:56.390Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"abc3d3fb-115e-4477-a6e1-bb9dd289ca1d","seconds":300}
{"level":"info","ts":"2026-11-03T15:12:57.358Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"a77b5943-1c64-4be1-8cbe-123a79b701a4"}
{"level":"info","ts":"2026-11-03T15:12:57.358Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"a77b5943-1c64-4be1-8cbe-123a79b701a4"}
{"level":"info","ts":"2026-11-03T15:12:57.358Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"a77b5943-1c64-4be1-8cbe-123a79b701a4","1 head service found":"long-running-2zj8b-head-svc"}
{"level":"info","ts":"2026-11-03T15:12:57.358Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"a77b5943-1c64-4be1-8cbe-123a79b701a4","Found 1 head Pod":"long-running-2zj8b-head-cklc6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:12:57.358Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"a77b5943-1c64-4be1-8cbe-123a79b701a4","head Pod":"long-running-2zj8b-head-cklc6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-2zj8b-head-cklc6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:12:57.358Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"a77b5943-1c64-4be1-8cbe-123a79b701a4","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:12:57.358Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"a77b5943-1c64-4be1-8cbe-123a79b701a4","worker Pod":"long-running-2zj8b-small-group-worker-mqllk","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-2zj8b-small-group-worker-mqllk. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:12:57.358Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"a77b5943-1c64-4be1-8cbe-123a79b701a4","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:12:57.358Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"a77b5943-1c64-4be1-8cbe-123a79b701a4","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:12:57.358Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"a77b5943-1c64-4be1-8cbe-123a79b701a4","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:12:57.358Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"a77b5943-1c64-4be1-8cbe-123a79b701a4","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:12:57.358Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"a77b5943-1c64-4be1-8cbe-123a79b701a4","podName":"long-running-2zj8b-head-cklc6","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:11:18Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:12:57.358Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"a77b5943-1c64-4be1-8cbe-123a79b701a4","podName":"long-running-2zj8b-head-cklc6","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:11:18Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:12:57.358Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"a77b5943-1c64-4be1-8cbe-123a79b701a4","name":"long-running-2zj8b","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:12:57Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:12:57.420Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"f7f0afe9-04a3-41b3-b73b-c3b995ff7cf2","cluster":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:12:57.420Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"784846e3-9adf-490c-b1b9-77fe69ffe248","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:12:57.420Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"784846e3-9adf-490c-b1b9-77fe69ffe248","name":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:12:57.420Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"784846e3-9adf-490c-b1b9-77fe69ffe248","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:12:57.421Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"784846e3-9adf-490c-b1b9-77fe69ffe248"}
{"level":"info","ts":"2026-11-03T15:12:57.421Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"784846e3-9adf-490c-b1b9-77fe69ffe248","RayCluster":"long-running-2zj8b","State":""}
{"level":"info","ts":"2026-11-03T15:12:57.421Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"784846e3-9adf-490c-b1b9-77fe69ffe248","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:11:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:12:57Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:12:57.421Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"784846e3-9adf-490c-b1b9-77fe69ffe248","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:12:57.482Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"2990ab2f-1fef-445b-9cc5-ecca17a7873b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:12:57.482Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"2990ab2f-1fef-445b-9cc5-ecca17a7873b","name":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:12:57.482Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"2990ab2f-1fef-445b-9cc5-ecca17a7873b","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:12:57.482Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"2990ab2f-1fef-445b-9cc5-ecca17a7873b"}
{"level":"info","ts":"2026-11-03T15:12:57.482Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"2990ab2f-1fef-445b-9cc5-ecca17a7873b","RayCluster":"long-running-2zj8b","State":""}
{"level":"info","ts":"2026-11-03T15:12:57.482Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"2990ab2f-1fef-445b-9cc5-ecca17a7873b","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:12:57Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:12:57Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:12:59.421Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"ee72bfe8-a6ca-4971-a90e-e2fc43ef32dc"}
{"level":"info","ts":"2026-11-03T15:12:59.421Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"ee72bfe8-a6ca-4971-a90e-e2fc43ef32dc"}
{"level":"info","ts":"2026-11-03T15:12:59.421Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"ee72bfe8-a6ca-4971-a90e-e2fc43ef32dc","1 head service found":"long-running-2zj8b-head-svc"}
{"level":"info","ts":"2026-11-03T15:12:59.421Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"ee72bfe8-a6ca-4971-a90e-e2fc43ef32dc","Found 1 head Pod":"long-running-2zj8b-head-cklc6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:12:59.421Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"ee72bfe8-a6ca-4971-a90e-e2fc43ef32dc","head Pod":"long-running-2zj8b-head-cklc6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-2zj8b-head-cklc6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:12:59.421Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"ee72bfe8-a6ca-4971-a90e-e2fc43ef32dc","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:12:59.421Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"ee72bfe8-a6ca-4971-a90e-e2fc43ef32dc","worker Pod":"long-running-2zj8b-small-group-worker-mqllk","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-2zj8b-small-group-worker-mqllk. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:12:59.421Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"ee72bfe8-a6ca-4971-a90e-e2fc43ef32dc","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:12:59.421Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"ee72bfe8-a6ca-4971-a90e-e2fc43ef32dc","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:12:59.421Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"ee72bfe8-a6ca-4971-a90e-e2fc43ef32dc","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:12:59.422Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"ee72bfe8-a6ca-4971-a90e-e2fc43ef32dc","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:12:59.422Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"ee72bfe8-a6ca-4971-a90e-e2fc43ef32dc","podName":"long-running-2zj8b-head-cklc6","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:11:18Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:12:59.422Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"ee72bfe8-a6ca-4971-a90e-e2fc43ef32dc","podName":"long-running-2zj8b-head-cklc6","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:11:18Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:12:59.422Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"ee72bfe8-a6ca-4971-a90e-e2fc43ef32dc","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:12:59.422Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"ee72bfe8-a6ca-4971-a90e-e2fc43ef32dc","seconds":300}
{"level":"info","ts":"2026-11-03T15:13:00.482Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"ed5fb83e-9ba1-4d75-809e-1d6dc8669356","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:13:00.482Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"ed5fb83e-9ba1-4d75-809e-1d6dc8669356","name":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:13:00.482Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"ed5fb83e-9ba1-4d75-809e-1d6dc8669356","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:13:00.483Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"ed5fb83e-9ba1-4d75-809e-1d6dc8669356"}
{"level":"info","ts":"2026-11-03T15:13:00.483Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"ed5fb83e-9ba1-4d75-809e-1d6dc8669356","RayCluster":"long-running-2zj8b","State":""}
{"level":"info","ts":"2026-11-03T15:13:00.483Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"ed5fb83e-9ba1-4d75-809e-1d6dc8669356","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:12:57Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:12:57Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:13:01.221Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"a9dd1cd4-3062-4cb2-b1f1-32b457e6f0e2"}
{"level":"info","ts":"2026-11-03T15:13:01.221Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"a9dd1cd4-3062-4cb2-b1f1-32b457e6f0e2"}
{"level":"info","ts":"2026-11-03T15:13:01.221Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"a9dd1cd4-3062-4cb2-b1f1-32b457e6f0e2","1 head service found":"long-running-2zj8b-head-svc"}
{"level":"info","ts":"2026-11-03T15:13:01.221Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"a9dd1cd4-3062-4cb2-b1f1-32b457e6f0e2","Found 1 head Pod":"long-running-2zj8b-head-cklc6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:13:01.222Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"a9dd1cd4-3062-4cb2-b1f1-32b457e6f0e2","head Pod":"long-running-2zj8b-head-cklc6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-2zj8b-head-cklc6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:13:01.222Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"a9dd1cd4-3062-4cb2-b1f1-32b457e6f0e2","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:13:01.222Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"a9dd1cd4-3062-4cb2-b1f1-32b457e6f0e2","worker Pod":"long-running-2zj8b-small-group-worker-mqllk","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-2zj8b-small-group-worker-mqllk. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:13:01.222Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"a9dd1cd4-3062-4cb2-b1f1-32b457e6f0e2","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:13:01.222Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"a9dd1cd4-3062-4cb2-b1f1-32b457e6f0e2","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:13:01.222Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"a9dd1cd4-3062-4cb2-b1f1-32b457e6f0e2","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:13:01.222Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"a9dd1cd4-3062-4cb2-b1f1-32b457e6f0e2","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:13:01.222Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"a9dd1cd4-3062-4cb2-b1f1-32b457e6f0e2","podName":"long-running-2zj8b-head-cklc6","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:11:18Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:13:01.222Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"a9dd1cd4-3062-4cb2-b1f1-32b457e6f0e2","podName":"long-running-2zj8b-head-cklc6","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:11:18Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:13:01.222Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"a9dd1cd4-3062-4cb2-b1f1-32b457e6f0e2","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:13:01.222Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"a9dd1cd4-3062-4cb2-b1f1-32b457e6f0e2","seconds":300}
{"level":"info","ts":"2026-11-03T15:13:02.283Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"0a033c0d-651b-4b1f-af82-228c0638c94f"}
{"level":"info","ts":"2026-11-03T15:13:02.283Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"0a033c0d-651b-4b1f-af82-228c0638c94f"}
{"level":"info","ts":"2026-11-03T15:13:02.283Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"0a033c0d-651b-4b1f-af82-228c0638c94f","1 head service found":"long-running-2zj8b-head-svc"}
{"level":"info","ts":"2026-11-03T15:13:02.283Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"0a033c0d-651b-4b1f-af82-228c0638c94f","Found 1 head Pod":"long-running-2zj8b-head-cklc6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:13:02.283Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"0a033c0d-651b-4b1f-af82-228c0638c94f","head Pod":"long-running-2zj8b-head-cklc6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-2zj8b-head-cklc6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:13:02.283Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"0a033c0d-651b-4b1f-af82-228c0638c94f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:13:02.283Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"0a033c0d-651b-4b1f-af82-228c0638c94f","worker Pod":"long-running-2zj8b-small-group-worker-mqllk","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-2zj8b-small-group-worker-mqllk. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:13:02.283Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"0a033c0d-651b-4b1f-af82-228c0638c94f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:13:02.283Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"0a033c0d-651b-4b1f-af82-228c0638c94f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:13:02.283Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"0a033c0d-651b-4b1f-af82-228c0638c94f","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:13:02.283Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"0a033c0d-651b-4b1f-af82-228c0638c94f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:13:02.283Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"0a033c0d-651b-4b1f-af82-228c0638c94f","podName":"long-running-2zj8b-head-cklc6","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:11:18Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:13:02.283Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"0a033c0d-651b-4b1f-af82-228c0638c94f","podName":"long-running-2zj8b-head-cklc6","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:11:18Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:13:02.283Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"0a033c0d-651b-4b1f-af82-228c0638c94f","name":"long-running-2zj8b","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:12:57Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:13:02.344Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"f67fae40-4d67-4e61-b294-6de7245087e2","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:13:02.344Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"f67fae40-4d67-4e61-b294-6de7245087e2","name":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:13:02.344Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"8ad843a7-25fb-4706-baac-4ad413756c07","cluster":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:13:02.344Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"f67fae40-4d67-4e61-b294-6de7245087e2","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:13:02.345Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"f67fae40-4d67-4e61-b294-6de7245087e2"}
{"level":"info","ts":"2026-11-03T15:13:02.345Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"f67fae40-4d67-4e61-b294-6de7245087e2","RayCluster":"long-running-2zj8b","State":""}
{"level":"info","ts":"2026-11-03T15:13:02.345Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"f67fae40-4d67-4e61-b294-6de7245087e2","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:12:57Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:13:02.345Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"f67fae40-4d67-4e61-b294-6de7245087e2","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:13:02.406Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"51aeb0eb-f40f-4b3e-b975-8622c35996d0","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:13:02.406Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"51aeb0eb-f40f-4b3e-b975-8622c35996d0","name":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:13:02.407Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"51aeb0eb-f40f-4b3e-b975-8622c35996d0","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:13:02.407Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"51aeb0eb-f40f-4b3e-b975-8622c35996d0"}
{"level":"info","ts":"2026-11-03T15:13:02.407Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"51aeb0eb-f40f-4b3e-b975-8622c35996d0","RayCluster":"long-running-2zj8b","State":""}
{"level":"info","ts":"2026-11-03T15:13:02.407Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"51aeb0eb-f40f-4b3e-b975-8622c35996d0","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:13:04.345Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"e0625e58-cf52-46c1-aec3-58a154a9e282"}
{"level":"info","ts":"2026-11-03T15:13:04.345Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"e0625e58-cf52-46c1-aec3-58a154a9e282"}
{"level":"info","ts":"2026-11-03T15:13:04.345Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"e0625e58-cf52-46c1-aec3-58a154a9e282","1 head service found":"long-running-2zj8b-head-svc"}
{"level":"info","ts":"2026-11-03T15:13:04.345Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"e0625e58-cf52-46c1-aec3-58a154a9e282","Found 1 head Pod":"long-running-2zj8b-head-cklc6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:13:04.345Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"e0625e58-cf52-46c1-aec3-58a154a9e282","head Pod":"long-running-2zj8b-head-cklc6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-2zj8b-head-cklc6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:13:04.345Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"e0625e58-cf52-46c1-aec3-58a154a9e282","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:13:04.345Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"e0625e58-cf52-46c1-aec3-58a154a9e282","worker Pod":"long-running-2zj8b-small-group-worker-mqllk","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-2zj8b-small-group-worker-mqllk. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:13:04.345Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"e0625e58-cf52-46c1-aec3-58a154a9e282","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:13:04.345Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"e0625e58-cf52-46c1-aec3-58a154a9e282","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:13:04.345Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"e0625e58-cf52-46c1-aec3-58a154a9e282","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:13:04.345Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"e0625e58-cf52-46c1-aec3-58a154a9e282","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:13:04.346Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"e0625e58-cf52-46c1-aec3-58a154a9e282","podName":"long-running-2zj8b-head-cklc6","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:11:18Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:13:04.346Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"e0625e58-cf52-46c1-aec3-58a154a9e282","podName":"long-running-2zj8b-head-cklc6","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:11:18Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:13:04.346Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"e0625e58-cf52-46c1-aec3-58a154a9e282","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:13:04.346Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"e0625e58-cf52-46c1-aec3-58a154a9e282","seconds":300}
{"level":"info","ts":"2026-11-03T15:13:05.408Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"c4e56225-fc0c-4095-9851-d910bfcc5a48","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:13:05.408Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"c4e56225-fc0c-4095-9851-d910bfcc5a48","name":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:13:05.408Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"c4e56225-fc0c-4095-9851-d910bfcc5a48","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:13:05.408Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"c4e56225-fc0c-4095-9851-d910bfcc5a48"}
{"level":"info","ts":"2026-11-03T15:13:05.408Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"c4e56225-fc0c-4095-9851-d910bfcc5a48","RayCluster":"long-running-2zj8b","State":""}
{"level":"info","ts":"2026-11-03T15:13:05.408Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"c4e56225-fc0c-4095-9851-d910bfcc5a48","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:13:08.409Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"77050329-93f3-4e47-af4a-16c446bc2da4","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:13:08.409Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"77050329-93f3-4e47-af4a-16c446bc2da4","name":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:13:08.409Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"77050329-93f3-4e47-af4a-16c446bc2da4","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:13:08.409Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"77050329-93f3-4e47-af4a-16c446bc2da4"}
{"level":"info","ts":"2026-11-03T15:13:08.409Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"77050329-93f3-4e47-af4a-16c446bc2da4","RayCluster":"long-running-2zj8b","State":""}
{"level":"info","ts":"2026-11-03T15:13:08.409Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"77050329-93f3-4e47-af4a-16c446bc2da4","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:13:08.412Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"42c5ac65-a75f-494b-9a11-93f03cda29ad"}
{"level":"info","ts":"2026-11-03T15:13:08.412Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"42c5ac65-a75f-494b-9a11-93f03cda29ad"}
{"level":"info","ts":"2026-11-03T15:13:08.412Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"42c5ac65-a75f-494b-9a11-93f03cda29ad","1 head service found":"long-running-2zj8b-head-svc"}
{"level":"info","ts":"2026-11-03T15:13:08.412Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"42c5ac65-a75f-494b-9a11-93f03cda29ad","Found 1 head Pod":"long-running-2zj8b-head-cklc6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:13:08.412Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"42c5ac65-a75f-494b-9a11-93f03cda29ad","head Pod":"long-running-2zj8b-head-cklc6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-2zj8b-head-cklc6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:13:08.412Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"42c5ac65-a75f-494b-9a11-93f03cda29ad","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:13:08.412Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"42c5ac65-a75f-494b-9a11-93f03cda29ad","worker Pod":"long-running-2zj8b-small-group-worker-mqllk","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-2zj8b-small-group-worker-mqllk. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:13:08.412Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"42c5ac65-a75f-494b-9a11-93f03cda29ad","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:13:08.412Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"42c5ac65-a75f-494b-9a11-93f03cda29ad","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:13:08.412Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"42c5ac65-a75f-494b-9a11-93f03cda29ad","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:13:08.412Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"42c5ac65-a75f-494b-9a11-93f03cda29ad","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:13:08.413Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"42c5ac65-a75f-494b-9a11-93f03cda29ad","podName":"long-running-2zj8b-small-group-worker-mqllk","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:11:18Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:13:08.413Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"42c5ac65-a75f-494b-9a11-93f03cda29ad","podName":"long-running-2zj8b-small-group-worker-mqllk","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:11:18Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:13:08.413Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"42c5ac65-a75f-494b-9a11-93f03cda29ad","name":"long-running-2zj8b","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:08Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:13:08Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:13:08.474Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"336236a3-196f-4c92-83e4-2654b42602d6","cluster":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:13:08.474Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"db320f3f-f2c4-4378-b618-62ff87a14751","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:13:08.474Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"db320f3f-f2c4-4378-b618-62ff87a14751","name":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:13:08.474Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"db320f3f-f2c4-4378-b618-62ff87a14751","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:13:08.474Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"db320f3f-f2c4-4378-b618-62ff87a14751"}
{"level":"info","ts":"2026-11-03T15:13:08.474Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"db320f3f-f2c4-4378-b618-62ff87a14751","RayCluster":"long-running-2zj8b","State":""}
{"level":"info","ts":"2026-11-03T15:13:08.474Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"db320f3f-f2c4-4378-b618-62ff87a14751","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:08Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:13:08Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:13:08.474Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"db320f3f-f2c4-4378-b618-62ff87a14751","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:13:08.535Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"2ff7550d-7d93-4cbd-ae9a-4dc453d6d91e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:13:08.535Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"2ff7550d-7d93-4cbd-ae9a-4dc453d6d91e","name":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:13:08.535Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"2ff7550d-7d93-4cbd-ae9a-4dc453d6d91e","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:13:08.535Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"2ff7550d-7d93-4cbd-ae9a-4dc453d6d91e"}
{"level":"info","ts":"2026-11-03T15:13:08.535Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"2ff7550d-7d93-4cbd-ae9a-4dc453d6d91e","RayCluster":"long-running-2zj8b","State":""}
{"level":"info","ts":"2026-11-03T15:13:08.535Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"2ff7550d-7d93-4cbd-ae9a-4dc453d6d91e","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:08Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:13:08Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:08Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:13:08Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:13:10.475Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"5d475867-0133-457a-bbaf-d33ba1baf805"}
{"level":"info","ts":"2026-11-03T15:13:10.475Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"5d475867-0133-457a-bbaf-d33ba1baf805"}
{"level":"info","ts":"2026-11-03T15:13:10.475Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"5d475867-0133-457a-bbaf-d33ba1baf805","1 head service found":"long-running-2zj8b-head-svc"}
{"level":"info","ts":"2026-11-03T15:13:10.475Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"5d475867-0133-457a-bbaf-d33ba1baf805","Found 1 head Pod":"long-running-2zj8b-head-cklc6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:13:10.475Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"5d475867-0133-457a-bbaf-d33ba1baf805","head Pod":"long-running-2zj8b-head-cklc6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-2zj8b-head-cklc6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:13:10.475Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"5d475867-0133-457a-bbaf-d33ba1baf805","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:13:10.475Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"5d475867-0133-457a-bbaf-d33ba1baf805","worker Pod":"long-running-2zj8b-small-group-worker-mqllk","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-2zj8b-small-group-worker-mqllk. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:13:10.475Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"5d475867-0133-457a-bbaf-d33ba1baf805","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:13:10.475Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"5d475867-0133-457a-bbaf-d33ba1baf805","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:13:10.475Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"5d475867-0133-457a-bbaf-d33ba1baf805","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:13:10.475Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"5d475867-0133-457a-bbaf-d33ba1baf805","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:13:10.475Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"5d475867-0133-457a-bbaf-d33ba1baf805","podName":"long-running-2zj8b-small-group-worker-mqllk","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:11:18Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:13:10.475Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"5d475867-0133-457a-bbaf-d33ba1baf805","podName":"long-running-2zj8b-small-group-worker-mqllk","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:11:18Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:13:10.475Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"5d475867-0133-457a-bbaf-d33ba1baf805","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:13:10.475Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"5d475867-0133-457a-bbaf-d33ba1baf805","seconds":300}
{"level":"info","ts":"2026-11-03T15:13:11.536Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"43b3a9bf-168d-49f1-b59d-e3c1acd4fcbd","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:13:11.536Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"43b3a9bf-168d-49f1-b59d-e3c1acd4fcbd","name":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:13:11.536Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"43b3a9bf-168d-49f1-b59d-e3c1acd4fcbd","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:13:11.536Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"43b3a9bf-168d-49f1-b59d-e3c1acd4fcbd"}
{"level":"info","ts":"2026-11-03T15:13:11.536Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"43b3a9bf-168d-49f1-b59d-e3c1acd4fcbd","RayCluster":"long-running-2zj8b","State":""}
{"level":"info","ts":"2026-11-03T15:13:11.536Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"43b3a9bf-168d-49f1-b59d-e3c1acd4fcbd","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:08Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:13:08Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:08Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:13:08Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:13:13.245Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"8233b25c-0479-4079-b7cb-3ac0c9dddf01"}
{"level":"info","ts":"2026-11-03T15:13:13.245Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"8233b25c-0479-4079-b7cb-3ac0c9dddf01"}
{"level":"info","ts":"2026-11-03T15:13:13.245Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"8233b25c-0479-4079-b7cb-3ac0c9dddf01","1 head service found":"long-running-2zj8b-head-svc"}
{"level":"info","ts":"2026-11-03T15:13:13.245Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"8233b25c-0479-4079-b7cb-3ac0c9dddf01","Found 1 head Pod":"long-running-2zj8b-head-cklc6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:13:13.245Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"8233b25c-0479-4079-b7cb-3ac0c9dddf01","head Pod":"long-running-2zj8b-head-cklc6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-2zj8b-head-cklc6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:13:13.245Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"8233b25c-0479-4079-b7cb-3ac0c9dddf01","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:13:13.245Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"8233b25c-0479-4079-b7cb-3ac0c9dddf01","worker Pod":"long-running-2zj8b-small-group-worker-mqllk","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-2zj8b-small-group-worker-mqllk. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:13:13.245Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"8233b25c-0479-4079-b7cb-3ac0c9dddf01","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:13:13.245Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"8233b25c-0479-4079-b7cb-3ac0c9dddf01","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:13:13.245Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"8233b25c-0479-4079-b7cb-3ac0c9dddf01","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:13:13.245Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"8233b25c-0479-4079-b7cb-3ac0c9dddf01","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:13:13.245Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"8233b25c-0479-4079-b7cb-3ac0c9dddf01","name":"long-running-2zj8b","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:08Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:13:08Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:13Z","stateTransitionTimes":{"ready":"2026-11-03T15:13:13Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:13:08Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:13:13Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:13:13.308Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"0819a1d0-ada7-481e-a02d-20c07742ae04","cluster":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:13:13.308Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"3a9fec4f-a6da-474b-8d0d-44eccc425fc7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:13:13.308Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"3a9fec4f-a6da-474b-8d0d-44eccc425fc7","name":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:13:13.308Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"3a9fec4f-a6da-474b-8d0d-44eccc425fc7","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:13:13.308Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"3a9fec4f-a6da-474b-8d0d-44eccc425fc7"}
{"level":"info","ts":"2026-11-03T15:13:13.371Z","logger":"controllers.RayJob","msg":"Created submitter Kubernetes Job for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"3a9fec4f-a6da-474b-8d0d-44eccc425fc7","Kubernetes Job":"long-running"}
{"level":"info","ts":"2026-11-03T15:13:13.371Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"3a9fec4f-a6da-474b-8d0d-44eccc425fc7","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:08Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:13:08Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","dashboardURL":"long-running-2zj8b-head-svc.test-ns-5lqjm.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:08Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:13:08Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:13:13.371Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"3a9fec4f-a6da-474b-8d0d-44eccc425fc7","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-11-03T15:13:13.431Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"cea7975c-bce4-4fac-83d9-6b9968e960ac","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:13:13.432Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"cea7975c-bce4-4fac-83d9-6b9968e960ac","name":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:13:13.432Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"cea7975c-bce4-4fac-83d9-6b9968e960ac","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:13:13.432Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"cea7975c-bce4-4fac-83d9-6b9968e960ac"}
{"level":"info","ts":"2026-11-03T15:13:13.432Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job for RayJob already exists","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"cea7975c-bce4-4fac-83d9-6b9968e960ac","Kubernetes Job":"long-running"}
{"level":"info","ts":"2026-11-03T15:13:13.432Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"cea7975c-bce4-4fac-83d9-6b9968e960ac","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:08Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:13:08Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","dashboardURL":"long-running-2zj8b-head-svc.test-ns-5lqjm.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:08Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:13:08Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:13:13.432Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"cea7975c-bce4-4fac-83d9-6b9968e960ac","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-11-03T15:13:13.487Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"cea7975c-bce4-4fac-83d9-6b9968e960ac","error":"Operation cannot be fulfilled on rayjobs.ray.io \"long-running\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-11-03T15:13:13.487Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"cea7975c-bce4-4fac-83d9-6b9968e960ac"}
{"level":"error","ts":"2026-11-03T15:13:13.487Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"cea7975c-bce4-4fac-83d9-6b9968e960ac","error":"Operation cannot be fulfilled on rayjobs.ray.io \"long-running\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:13:13.487Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"8b29a980-19fc-42e2-a298-63ab6c6ddaa1","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:13:13.487Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"8b29a980-19fc-42e2-a298-63ab6c6ddaa1","name":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:13:13.487Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"8b29a980-19fc-42e2-a298-63ab6c6ddaa1","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:13:13.487Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"8b29a980-19fc-42e2-a298-63ab6c6ddaa1"}
{"level":"info","ts":"2026-11-03T15:13:13.487Z","logger":"controllers.RayJob","msg":"Create a Kubernetes Service","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"8b29a980-19fc-42e2-a298-63ab6c6ddaa1","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:13:13.549Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"98cf9f9f-4e8e-4079-acae-8ab6e753e90e","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:13:13.549Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"98cf9f9f-4e8e-4079-acae-8ab6e753e90e","name":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:13:13.549Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"98cf9f9f-4e8e-4079-acae-8ab6e753e90e","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:13:13.549Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"98cf9f9f-4e8e-4079-acae-8ab6e753e90e"}
{"level":"info","ts":"2026-11-03T15:13:13.549Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"98cf9f9f-4e8e-4079-acae-8ab6e753e90e","rayCluster":"long-running-2zj8b","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:13:13.557Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"6f1aff18-d61a-4329-80c3-22e99e838ac7","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:13:13.557Z","logger":"controllers.RayJob","msg":"Try to transition the status to `Suspending`","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"6f1aff18-d61a-4329-80c3-22e99e838ac7","oldStatus":"Running"}
{"level":"info","ts":"2026-11-03T15:13:13.557Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"6f1aff18-d61a-4329-80c3-22e99e838ac7","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","dashboardURL":"long-running-2zj8b-head-svc.test-ns-5lqjm.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:08Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:13:08Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","dashboardURL":"long-running-2zj8b-head-svc.test-ns-5lqjm.svc.cluster.local:8265","jobDeploymentStatus":"Suspending","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:08Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:13:08Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:13:13.557Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"6f1aff18-d61a-4329-80c3-22e99e838ac7","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Suspending"}
{"level":"info","ts":"2026-11-03T15:13:13.619Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"37a9f2f3-1ab7-4b22-858e-1a92d005bf3b","JobStatus":"","JobDeploymentStatus":"Suspending","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:13:13.676Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob is deleted","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"37a9f2f3-1ab7-4b22-858e-1a92d005bf3b","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:13:13.676Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"37a9f2f3-1ab7-4b22-858e-1a92d005bf3b","isClusterDeleted":false}
{"level":"info","ts":"2026-11-03T15:13:13.678Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"reconcileID":"2b4066f9-a665-4b44-9766-b25f4df39def"}
{"level":"info","ts":"2026-11-03T15:13:13.678Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"long-running-2zj8b","reconcileID":"5e7ec611-4ca2-46bc-87da-b532a21dbe29","namespacedName":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:13:13.678Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"long-running-2zj8b","reconcileID":"5e7ec611-4ca2-46bc-87da-b532a21dbe29","cluster":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:13:13.678Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"long-running-2zj8b","reconcileID":"5e7ec611-4ca2-46bc-87da-b532a21dbe29","cluster":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:13:13.678Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"long-running-2zj8b","reconcileID":"5e7ec611-4ca2-46bc-87da-b532a21dbe29","namespace":"test-ns-5lqjm","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-11-03T15:13:13.728Z","logger":"controllers.RayJob","msg":"The associated submitter Kubernetes Job for RayJob is deleted","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"37a9f2f3-1ab7-4b22-858e-1a92d005bf3b","Submitter K8s Job":"long-running"}
{"level":"info","ts":"2026-11-03T15:13:13.728Z","logger":"controllers.RayJob","msg":"deleteSubmitterJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"37a9f2f3-1ab7-4b22-858e-1a92d005bf3b","isJobDeleted":false}
{"level":"info","ts":"2026-11-03T15:13:13.728Z","logger":"controllers.RayJob","msg":"The release of the compute resources has not been completed yet. Wait for the resources to be deleted before the status transitions to avoid a resource leak.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"37a9f2f3-1ab7-4b22-858e-1a92d005bf3b"}
{"level":"info","ts":"2026-11-03T15:13:13.728Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"80bfcda6-49da-4d58-9902-e2003e08577b","JobStatus":"","JobDeploymentStatus":"Suspending","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:13:13.728Z","logger":"controllers.RayJob","msg":"The deletion of the associated RayCluster for RayJob is ongoing.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"80bfcda6-49da-4d58-9902-e2003e08577b","RayCluster":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:13:13.728Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"80bfcda6-49da-4d58-9902-e2003e08577b","isClusterDeleted":false}
{"level":"info","ts":"2026-11-03T15:13:13.728Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job has been already deleted","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"80bfcda6-49da-4d58-9902-e2003e08577b","Kubernetes Job":""}
{"level":"info","ts":"2026-11-03T15:13:13.728Z","logger":"controllers.RayJob","msg":"deleteSubmitterJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"80bfcda6-49da-4d58-9902-e2003e08577b","isJobDeleted":true}
{"level":"info","ts":"2026-11-03T15:13:13.728Z","logger":"controllers.RayJob","msg":"The release of the compute resources has not been completed yet. Wait for the resources to be deleted before the status transitions to avoid a resource leak.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"80bfcda6-49da-4d58-9902-e2003e08577b"}
{"level":"info","ts":"2026-11-03T15:13:13.729Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"3f714cbc-9199-48c7-ba1e-db5c24978c38","JobStatus":"","JobDeploymentStatus":"Suspending","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:13:13.729Z","logger":"controllers.RayJob","msg":"The deletion of the associated RayCluster for RayJob is ongoing.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"3f714cbc-9199-48c7-ba1e-db5c24978c38","RayCluster":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:13:13.729Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"3f714cbc-9199-48c7-ba1e-db5c24978c38","isClusterDeleted":false}
{"level":"info","ts":"2026-11-03T15:13:13.729Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job has been already deleted","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"3f714cbc-9199-48c7-ba1e-db5c24978c38","Kubernetes Job":""}
{"level":"info","ts":"2026-11-03T15:13:13.729Z","logger":"controllers.RayJob","msg":"deleteSubmitterJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"3f714cbc-9199-48c7-ba1e-db5c24978c38","isJobDeleted":true}
{"level":"info","ts":"2026-11-03T15:13:13.729Z","logger":"controllers.RayJob","msg":"The release of the compute resources has not been completed yet. Wait for the resources to be deleted before the status transitions to avoid a resource leak.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"3f714cbc-9199-48c7-ba1e-db5c24978c38"}
{"level":"info","ts":"2026-11-03T15:13:13.731Z","logger":"authentication-controller","msg":"Deleted shared ReferenceGrant (last cluster with auth)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"long-running-2zj8b","reconcileID":"5e7ec611-4ca2-46bc-87da-b532a21dbe29","name":"kuberay-gateway-access","namespace":"test-ns-5lqjm"}
{"level":"info","ts":"2026-11-03T15:13:13.784Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"long-running-2zj8b","reconcileID":"5e7ec611-4ca2-46bc-87da-b532a21dbe29","name":"test-ns-5lqjm-long-running-2zj8b","namespace":"opendatahub"}
{"level":"info","ts":"2026-11-03T15:13:13.838Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"long-running-2zj8b","reconcileID":"5e7ec611-4ca2-46bc-87da-b532a21dbe29","configMap":"kube-rbac-proxy-config-long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:13:13.890Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"long-running-2zj8b","reconcileID":"5e7ec611-4ca2-46bc-87da-b532a21dbe29","serviceAccount":"long-running-2zj8b-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-11-03T15:13:14.056Z","logger":"raycluster-resource","msg":"default","name":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:13:14.056Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"long-running-2zj8b","namespace":"test-ns-5lqjm"}
{"level":"info","ts":"2026-11-03T15:13:14.219Z","logger":"raycluster-resource","msg":"validate update","name":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:13:14.274Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"long-running-2zj8b","reconcileID":"5e7ec611-4ca2-46bc-87da-b532a21dbe29","cluster":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:13:14.274Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"long-running-2zj8b","reconcileID":"14a0d11c-442f-4c0d-ae1d-19e29108eda8","namespacedName":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:13:14.274Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"long-running-2zj8b","reconcileID":"14a0d11c-442f-4c0d-ae1d-19e29108eda8","cluster":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:13:14.274Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"long-running-2zj8b","reconcileID":"14a0d11c-442f-4c0d-ae1d-19e29108eda8","cluster":"long-running-2zj8b"}
{"level":"info","ts":"2026-11-03T15:13:14.274Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"long-running-2zj8b","reconcileID":"14a0d11c-442f-4c0d-ae1d-19e29108eda8","namespace":"test-ns-5lqjm","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-11-03T15:13:14.274Z","logger":"authentication-controller","msg":"ReferenceGrant already deleted","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"long-running-2zj8b","reconcileID":"14a0d11c-442f-4c0d-ae1d-19e29108eda8","name":"kuberay-gateway-access"}
{"level":"info","ts":"2026-11-03T15:13:14.274Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"d00469e8-d1db-4913-bc02-2268f8faeafa","JobStatus":"","JobDeploymentStatus":"Suspending","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:13:14.274Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"d00469e8-d1db-4913-bc02-2268f8faeafa","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:13:14.274Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"d00469e8-d1db-4913-bc02-2268f8faeafa","isClusterDeleted":true}
{"level":"info","ts":"2026-11-03T15:13:14.274Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job has been already deleted","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"d00469e8-d1db-4913-bc02-2268f8faeafa","Kubernetes Job":""}
{"level":"info","ts":"2026-11-03T15:13:14.274Z","logger":"controllers.RayJob","msg":"deleteSubmitterJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"d00469e8-d1db-4913-bc02-2268f8faeafa","isJobDeleted":true}
{"level":"info","ts":"2026-11-03T15:13:14.274Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"d00469e8-d1db-4913-bc02-2268f8faeafa","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-r9gbd","rayClusterName":"long-running-2zj8b","dashboardURL":"long-running-2zj8b-head-svc.test-ns-5lqjm.svc.cluster.local:8265","jobDeploymentStatus":"Suspending","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:08Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.35","serviceIP":"10.132.0.35","podName":"long-running-2zj8b-head-cklc6","serviceName":"long-running-2zj8b-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:13:08Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:11:12Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:11:13Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobDeploymentStatus":"Suspended","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:13:14.275Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"d00469e8-d1db-4913-bc02-2268f8faeafa","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Suspending","new JobDeploymentStatus":"Suspended"}
{"level":"error","ts":"2026-11-03T15:13:14.325Z","logger":"authentication-controller","msg":"Failed to remove finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"long-running-2zj8b","reconcileID":"14a0d11c-442f-4c0d-ae1d-19e29108eda8","error":"rayclusters.ray.io \"long-running-2zj8b\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleDeletion\n\t/workspace/controllers/ray/authentication_controller.go:195\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:13:14.325Z","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"long-running-2zj8b","reconcileID":"14a0d11c-442f-4c0d-ae1d-19e29108eda8"}
{"level":"error","ts":"2026-11-03T15:13:14.325Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"long-running-2zj8b","reconcileID":"14a0d11c-442f-4c0d-ae1d-19e29108eda8","error":"rayclusters.ray.io \"long-running-2zj8b\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:13:14.325Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"long-running-2zj8b","reconcileID":"3a318450-a0ca-4141-94b1-d1033c6f8d22","namespacedName":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:13:14.325Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"long-running-2zj8b","reconcileID":"3a318450-a0ca-4141-94b1-d1033c6f8d22"}
{"level":"info","ts":"2026-11-03T15:13:14.325Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"long-running-2zj8b","reconcileID":"3a318450-a0ca-4141-94b1-d1033c6f8d22","namespace":"test-ns-5lqjm","authClusters":0}
{"level":"info","ts":"2026-11-03T15:13:14.325Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-2zj8b","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"long-running-2zj8b","reconcileID":"3a318450-a0ca-4141-94b1-d1033c6f8d22","namespace":"test-ns-5lqjm"}
{"level":"info","ts":"2026-11-03T15:13:14.359Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"e7507501-4a8b-4f4f-aef0-19b2c6477191","JobStatus":"","JobDeploymentStatus":"Suspended","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:13:14.648Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"8138d786-eeba-40ac-a5af-4c8cce4849f3","JobStatus":"","JobDeploymentStatus":"Suspended","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:13:14.648Z","logger":"controllers.RayJob","msg":"The status is 'Suspended', but the suspend flag is false. Transition the status to 'New'.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"8138d786-eeba-40ac-a5af-4c8cce4849f3"}
{"level":"info","ts":"2026-11-03T15:13:14.648Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"8138d786-eeba-40ac-a5af-4c8cce4849f3","oldRayJobStatus":{"rayJobInfo":{},"jobDeploymentStatus":"Suspended","startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:13:14.648Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"8138d786-eeba-40ac-a5af-4c8cce4849f3","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Suspended","new JobDeploymentStatus":""}
{"level":"info","ts":"2026-11-03T15:13:14.707Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"eb7613f6-d1c3-4afc-b7f6-b8cc163b7697","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:13:14.707Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"eb7613f6-d1c3-4afc-b7f6-b8cc163b7697"}
{"level":"info","ts":"2026-11-03T15:13:14.707Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"eb7613f6-d1c3-4afc-b7f6-b8cc163b7697","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-11-03T15:13:14.707Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"eb7613f6-d1c3-4afc-b7f6-b8cc163b7697","oldRayJobStatus":{"rayJobInfo":{},"startTime":"2026-11-03T15:11:12Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:13:14.707Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"eb7613f6-d1c3-4afc-b7f6-b8cc163b7697","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:13:14.766Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"b3b33593-ed90-4161-a299-a1b491d78332","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:13:14.766Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"b3b33593-ed90-4161-a299-a1b491d78332","name":"long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:13:14.766Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"b3b33593-ed90-4161-a299-a1b491d78332","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:13:14.766Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"b3b33593-ed90-4161-a299-a1b491d78332","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:13:14.816Z","logger":"raycluster-resource","msg":"default","name":"long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:13:14.817Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"long-running-7qq77","namespace":"test-ns-5lqjm"}
{"level":"info","ts":"2026-11-03T15:13:14.869Z","logger":"raycluster-resource","msg":"validate create","name":"long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:13:14.923Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"b3b33593-ed90-4161-a299-a1b491d78332","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:13:14.923Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"b3b33593-ed90-4161-a299-a1b491d78332"}
{"level":"info","ts":"2026-11-03T15:13:14.923Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"b3b33593-ed90-4161-a299-a1b491d78332","RayCluster":"long-running-7qq77","State":""}
{"level":"info","ts":"2026-11-03T15:13:14.923Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"b3b33593-ed90-4161-a299-a1b491d78332","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:13:14.923Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"fb2e9654-aa48-48d1-8c38-005c1a4716d7"}
{"level":"info","ts":"2026-11-03T15:13:14.923Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"fb2e9654-aa48-48d1-8c38-005c1a4716d7"}
{"level":"info","ts":"2026-11-03T15:13:14.923Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"long-running-7qq77","reconcileID":"4372a436-0546-4640-9507-2f3488093330","namespacedName":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:13:14.923Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"7d1d9a74-49e5-46fe-a9e2-15187cee92c3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:13:14.923Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"7d1d9a74-49e5-46fe-a9e2-15187cee92c3","name":"long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:13:14.923Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"long-running-7qq77","reconcileID":"4372a436-0546-4640-9507-2f3488093330","mode":"IntegratedOAuth","cluster":"long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:13:14.923Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"long-running-7qq77","reconcileID":"4372a436-0546-4640-9507-2f3488093330","cluster":"long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:13:14.923Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"long-running-7qq77","reconcileID":"4372a436-0546-4640-9507-2f3488093330","cluster":"long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:13:14.923Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"7d1d9a74-49e5-46fe-a9e2-15187cee92c3","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:13:14.923Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"7d1d9a74-49e5-46fe-a9e2-15187cee92c3"}
{"level":"info","ts":"2026-11-03T15:13:14.923Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"7d1d9a74-49e5-46fe-a9e2-15187cee92c3","RayCluster":"long-running-7qq77","State":""}
{"level":"info","ts":"2026-11-03T15:13:14.923Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"7d1d9a74-49e5-46fe-a9e2-15187cee92c3","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:13:14.924Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"fd0a9853-3157-4aff-81ad-2b6861b9b53f","cluster":"long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:13:14.924Z","logger":"controllers.NetworkPolicy","msg":"Reconciling NetworkPolicies for RayCluster","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"55b1407e-954a-4af0-ab1c-58790b68d364","cluster":"long-running-7qq77","namespace":"test-ns-5lqjm"}
{"level":"info","ts":"2026-11-03T15:13:14.976Z","logger":"raycluster-resource","msg":"default","name":"long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:13:14.976Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"long-running-7qq77","namespace":"test-ns-5lqjm"}
{"level":"info","ts":"2026-11-03T15:13:14.980Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"fb2e9654-aa48-48d1-8c38-005c1a4716d7","name":"long-running-7qq77-head-svc"}
{"level":"info","ts":"2026-11-03T15:13:14.980Z","logger":"controllers.NetworkPolicy","msg":"Successfully created NetworkPolicy","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"55b1407e-954a-4af0-ab1c-58790b68d364","name":"long-running-7qq77-head"}
{"level":"info","ts":"2026-11-03T15:13:14.980Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"fb2e9654-aa48-48d1-8c38-005c1a4716d7"}
{"level":"info","ts":"2026-11-03T15:13:15.027Z","logger":"raycluster-resource","msg":"validate update","name":"long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:13:15.029Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"fb2e9654-aa48-48d1-8c38-005c1a4716d7","error":"certificate ray-head-cert-long-running-7qq77 not found: certificates.cert-manager.io \"ray-head-cert-long-running-7qq77\" not found"}
{"level":"error","ts":"2026-11-03T15:13:15.029Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"fb2e9654-aa48-48d1-8c38-005c1a4716d7","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-7qq77 not found: certificates.cert-manager.io \"ray-head-cert-long-running-7qq77\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:13:15.029Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"fb2e9654-aa48-48d1-8c38-005c1a4716d7","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-7qq77 not found: certificates.cert-manager.io \"ray-head-cert-long-running-7qq77\" not found"}
{"level":"info","ts":"2026-11-03T15:13:15.029Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"fb2e9654-aa48-48d1-8c38-005c1a4716d7","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"long-running-7qq77","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:13:15.029Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"fb2e9654-aa48-48d1-8c38-005c1a4716d7","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"long-running-7qq77","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:13:15.029Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"fb2e9654-aa48-48d1-8c38-005c1a4716d7","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"long-running-7qq77","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:13:15.029Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"fb2e9654-aa48-48d1-8c38-005c1a4716d7","name":"long-running-7qq77","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:13:15.034Z","logger":"controllers.NetworkPolicy","msg":"Successfully created NetworkPolicy","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"55b1407e-954a-4af0-ab1c-58790b68d364","name":"long-running-7qq77-workers-small-group"}
{"level":"info","ts":"2026-11-03T15:13:15.034Z","logger":"controllers.NetworkPolicy","msg":"Successfully reconciled NetworkPolicies for RayCluster","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"55b1407e-954a-4af0-ab1c-58790b68d364","cluster":"long-running-7qq77","namespace":"test-ns-5lqjm"}
{"level":"info","ts":"2026-11-03T15:13:15.034Z","logger":"controllers.NetworkPolicy","msg":"Reconciling NetworkPolicies for RayCluster","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"2734d95e-056d-4621-9be0-a495d8db413e","cluster":"long-running-7qq77","namespace":"test-ns-5lqjm"}
{"level":"info","ts":"2026-11-03T15:13:15.081Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"d71f7ec6-7525-4afb-beba-3dfb2b154475","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:13:15.081Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"d71f7ec6-7525-4afb-beba-3dfb2b154475","name":"long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:13:15.081Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"d71f7ec6-7525-4afb-beba-3dfb2b154475","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:13:15.082Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"d71f7ec6-7525-4afb-beba-3dfb2b154475"}
{"level":"info","ts":"2026-11-03T15:13:15.082Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"d71f7ec6-7525-4afb-beba-3dfb2b154475","RayCluster":"long-running-7qq77","State":""}
{"level":"info","ts":"2026-11-03T15:13:15.082Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"d71f7ec6-7525-4afb-beba-3dfb2b154475","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:13:15.090Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"fb2e9654-aa48-48d1-8c38-005c1a4716d7","name":"long-running-7qq77","error":"Operation cannot be fulfilled on rayclusters.ray.io \"long-running-7qq77\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-5lqjm","name":"long-running-7qq77"}}
{"level":"info","ts":"2026-11-03T15:13:15.090Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"fb2e9654-aa48-48d1-8c38-005c1a4716d7"}
{"level":"error","ts":"2026-11-03T15:13:15.090Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"fb2e9654-aa48-48d1-8c38-005c1a4716d7","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-7qq77 not found: certificates.cert-manager.io \"ray-head-cert-long-running-7qq77\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:13:15.090Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"d592289e-d373-40a7-a845-b4e4fb9295d9"}
{"level":"info","ts":"2026-11-03T15:13:15.090Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"d592289e-d373-40a7-a845-b4e4fb9295d9"}
{"level":"info","ts":"2026-11-03T15:13:15.090Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"d592289e-d373-40a7-a845-b4e4fb9295d9","1 head service found":"long-running-7qq77-head-svc"}
{"level":"info","ts":"2026-11-03T15:13:15.090Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"d592289e-d373-40a7-a845-b4e4fb9295d9"}
{"level":"info","ts":"2026-11-03T15:13:15.133Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"long-running-7qq77","reconcileID":"4372a436-0546-4640-9507-2f3488093330","name":"long-running-7qq77-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-11-03T15:13:15.139Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"d592289e-d373-40a7-a845-b4e4fb9295d9","error":"certificate ray-head-cert-long-running-7qq77 not found: certificates.cert-manager.io \"ray-head-cert-long-running-7qq77\" not found"}
{"level":"error","ts":"2026-11-03T15:13:15.139Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"d592289e-d373-40a7-a845-b4e4fb9295d9","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-7qq77 not found: certificates.cert-manager.io \"ray-head-cert-long-running-7qq77\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:13:15.139Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"d592289e-d373-40a7-a845-b4e4fb9295d9","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-7qq77 not found: certificates.cert-manager.io \"ray-head-cert-long-running-7qq77\" not found"}
{"level":"info","ts":"2026-11-03T15:13:15.139Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"d592289e-d373-40a7-a845-b4e4fb9295d9","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"long-running-7qq77","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:13:15.139Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"d592289e-d373-40a7-a845-b4e4fb9295d9","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"long-running-7qq77","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:13:15.139Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"d592289e-d373-40a7-a845-b4e4fb9295d9","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"long-running-7qq77","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:13:15.139Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"d592289e-d373-40a7-a845-b4e4fb9295d9","name":"long-running-7qq77","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:13:15.144Z","logger":"controllers.NetworkPolicy","msg":"Successfully reconciled NetworkPolicies for RayCluster","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"2734d95e-056d-4621-9be0-a495d8db413e","cluster":"long-running-7qq77","namespace":"test-ns-5lqjm"}
{"level":"info","ts":"2026-11-03T15:13:15.185Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"long-running-7qq77","reconcileID":"4372a436-0546-4640-9507-2f3488093330","name":"kuberay-gateway-access","namespace":"test-ns-5lqjm","operation":"created"}
{"level":"info","ts":"2026-11-03T15:13:15.199Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"d592289e-d373-40a7-a845-b4e4fb9295d9"}
{"level":"error","ts":"2026-11-03T15:13:15.199Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"d592289e-d373-40a7-a845-b4e4fb9295d9","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-7qq77 not found: certificates.cert-manager.io \"ray-head-cert-long-running-7qq77\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:13:15.199Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"ea4598d9-0c48-4ce7-93c5-01e11c0804af","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:13:15.199Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"ea4598d9-0c48-4ce7-93c5-01e11c0804af","name":"long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:13:15.199Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"ea4598d9-0c48-4ce7-93c5-01e11c0804af","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:13:15.200Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"ea4598d9-0c48-4ce7-93c5-01e11c0804af"}
{"level":"info","ts":"2026-11-03T15:13:15.200Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"ea4598d9-0c48-4ce7-93c5-01e11c0804af","RayCluster":"long-running-7qq77","State":""}
{"level":"info","ts":"2026-11-03T15:13:15.200Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"ea4598d9-0c48-4ce7-93c5-01e11c0804af","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:13:15.200Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"ea4598d9-0c48-4ce7-93c5-01e11c0804af","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:13:15.209Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"fcf58c57-17c3-4f8f-a039-c38da38b1f7b"}
{"level":"info","ts":"2026-11-03T15:13:15.209Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"fcf58c57-17c3-4f8f-a039-c38da38b1f7b"}
{"level":"info","ts":"2026-11-03T15:13:15.209Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"fcf58c57-17c3-4f8f-a039-c38da38b1f7b","1 head service found":"long-running-7qq77-head-svc"}
{"level":"info","ts":"2026-11-03T15:13:15.209Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"fcf58c57-17c3-4f8f-a039-c38da38b1f7b"}
{"level":"info","ts":"2026-11-03T15:13:15.240Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"long-running-7qq77","reconcileID":"4372a436-0546-4640-9507-2f3488093330","name":"test-ns-5lqjm-long-running-7qq77","operation":"created"}
{"level":"info","ts":"2026-11-03T15:13:15.259Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"fcf58c57-17c3-4f8f-a039-c38da38b1f7b","error":"certificate ray-head-cert-long-running-7qq77 not found: certificates.cert-manager.io \"ray-head-cert-long-running-7qq77\" not found"}
{"level":"error","ts":"2026-11-03T15:13:15.259Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"fcf58c57-17c3-4f8f-a039-c38da38b1f7b","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-7qq77 not found: certificates.cert-manager.io \"ray-head-cert-long-running-7qq77\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:13:15.259Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"fcf58c57-17c3-4f8f-a039-c38da38b1f7b","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-7qq77 not found: certificates.cert-manager.io \"ray-head-cert-long-running-7qq77\" not found"}
{"level":"info","ts":"2026-11-03T15:13:15.259Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"fcf58c57-17c3-4f8f-a039-c38da38b1f7b","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"long-running-7qq77","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:13:15.259Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"fcf58c57-17c3-4f8f-a039-c38da38b1f7b","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"long-running-7qq77","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:13:15.259Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"fcf58c57-17c3-4f8f-a039-c38da38b1f7b","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"long-running-7qq77","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:13:15.259Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"fcf58c57-17c3-4f8f-a039-c38da38b1f7b"}
{"level":"error","ts":"2026-11-03T15:13:15.259Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"fcf58c57-17c3-4f8f-a039-c38da38b1f7b","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-7qq77 not found: certificates.cert-manager.io \"ray-head-cert-long-running-7qq77\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:13:15.260Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"d98636cf-2dca-4fad-8122-a350b9861467","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:13:15.260Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"d98636cf-2dca-4fad-8122-a350b9861467","name":"long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:13:15.260Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"d98636cf-2dca-4fad-8122-a350b9861467","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:13:15.260Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"d98636cf-2dca-4fad-8122-a350b9861467"}
{"level":"info","ts":"2026-11-03T15:13:15.260Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"d98636cf-2dca-4fad-8122-a350b9861467","RayCluster":"long-running-7qq77","State":""}
{"level":"info","ts":"2026-11-03T15:13:15.260Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"d98636cf-2dca-4fad-8122-a350b9861467","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:13:15.280Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"df5d7e6b-d811-4db4-993d-1354d6db9869"}
{"level":"info","ts":"2026-11-03T15:13:15.280Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"df5d7e6b-d811-4db4-993d-1354d6db9869"}
{"level":"info","ts":"2026-11-03T15:13:15.280Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"df5d7e6b-d811-4db4-993d-1354d6db9869","1 head service found":"long-running-7qq77-head-svc"}
{"level":"info","ts":"2026-11-03T15:13:15.280Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"df5d7e6b-d811-4db4-993d-1354d6db9869"}
{"level":"info","ts":"2026-11-03T15:13:15.292Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"long-running-7qq77","reconcileID":"4372a436-0546-4640-9507-2f3488093330","name":"kube-rbac-proxy-config-long-running-7qq77","operation":"created"}
{"level":"info","ts":"2026-11-03T15:13:15.330Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"df5d7e6b-d811-4db4-993d-1354d6db9869","error":"certificate ray-head-cert-long-running-7qq77 not found: certificates.cert-manager.io \"ray-head-cert-long-running-7qq77\" not found"}
{"level":"error","ts":"2026-11-03T15:13:15.330Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"df5d7e6b-d811-4db4-993d-1354d6db9869","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-7qq77 not found: certificates.cert-manager.io \"ray-head-cert-long-running-7qq77\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:13:15.330Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"df5d7e6b-d811-4db4-993d-1354d6db9869","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-7qq77 not found: certificates.cert-manager.io \"ray-head-cert-long-running-7qq77\" not found"}
{"level":"info","ts":"2026-11-03T15:13:15.330Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"df5d7e6b-d811-4db4-993d-1354d6db9869","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"long-running-7qq77","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:13:15.330Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"df5d7e6b-d811-4db4-993d-1354d6db9869","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"long-running-7qq77","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:13:15.330Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"df5d7e6b-d811-4db4-993d-1354d6db9869","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"long-running-7qq77","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:13:15.330Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"df5d7e6b-d811-4db4-993d-1354d6db9869"}
{"level":"error","ts":"2026-11-03T15:13:15.330Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"df5d7e6b-d811-4db4-993d-1354d6db9869","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-7qq77 not found: certificates.cert-manager.io \"ray-head-cert-long-running-7qq77\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-03T15:13:15.349Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"long-running-7qq77","reconcileID":"4372a436-0546-4640-9507-2f3488093330","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"long-running-7qq77\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:167\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:13:15.349Z","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"long-running-7qq77","reconcileID":"4372a436-0546-4640-9507-2f3488093330"}
{"level":"error","ts":"2026-11-03T15:13:15.349Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"long-running-7qq77","reconcileID":"4372a436-0546-4640-9507-2f3488093330","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"long-running-7qq77\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:13:15.349Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"long-running-7qq77","reconcileID":"a35a0cf9-0442-4c49-a1c5-e1090269fed8","namespacedName":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:13:15.349Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"long-running-7qq77","reconcileID":"a35a0cf9-0442-4c49-a1c5-e1090269fed8","mode":"IntegratedOAuth","cluster":"long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:13:15.349Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"long-running-7qq77","reconcileID":"a35a0cf9-0442-4c49-a1c5-e1090269fed8","cluster":"long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:13:15.371Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"15cd97f1-4469-4d9b-b146-d38b76a3afbe"}
{"level":"info","ts":"2026-11-03T15:13:15.371Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"15cd97f1-4469-4d9b-b146-d38b76a3afbe"}
{"level":"info","ts":"2026-11-03T15:13:15.372Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"15cd97f1-4469-4d9b-b146-d38b76a3afbe","1 head service found":"long-running-7qq77-head-svc"}
{"level":"info","ts":"2026-11-03T15:13:15.372Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"15cd97f1-4469-4d9b-b146-d38b76a3afbe"}
{"level":"info","ts":"2026-11-03T15:13:15.410Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"long-running-7qq77","reconcileID":"a35a0cf9-0442-4c49-a1c5-e1090269fed8","cluster":"long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:13:15.410Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"88be85c1-b196-44d9-b708-6493b4407839","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:13:15.410Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"88be85c1-b196-44d9-b708-6493b4407839","name":"long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:13:15.410Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"88be85c1-b196-44d9-b708-6493b4407839","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:13:15.411Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"88be85c1-b196-44d9-b708-6493b4407839"}
{"level":"info","ts":"2026-11-03T15:13:15.411Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"88be85c1-b196-44d9-b708-6493b4407839","RayCluster":"long-running-7qq77","State":""}
{"level":"info","ts":"2026-11-03T15:13:15.411Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"88be85c1-b196-44d9-b708-6493b4407839","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:13:15.411Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"88be85c1-b196-44d9-b708-6493b4407839","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:13:15.421Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"15cd97f1-4469-4d9b-b146-d38b76a3afbe","error":"certificate ray-head-cert-long-running-7qq77 not found: certificates.cert-manager.io \"ray-head-cert-long-running-7qq77\" not found"}
{"level":"error","ts":"2026-11-03T15:13:15.421Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"15cd97f1-4469-4d9b-b146-d38b76a3afbe","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-7qq77 not found: certificates.cert-manager.io \"ray-head-cert-long-running-7qq77\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:13:15.421Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"15cd97f1-4469-4d9b-b146-d38b76a3afbe","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-7qq77 not found: certificates.cert-manager.io \"ray-head-cert-long-running-7qq77\" not found"}
{"level":"info","ts":"2026-11-03T15:13:15.421Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"15cd97f1-4469-4d9b-b146-d38b76a3afbe","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"long-running-7qq77","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:13:15.421Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"15cd97f1-4469-4d9b-b146-d38b76a3afbe","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"long-running-7qq77","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:13:15.421Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"15cd97f1-4469-4d9b-b146-d38b76a3afbe","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"long-running-7qq77","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:13:15.421Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"15cd97f1-4469-4d9b-b146-d38b76a3afbe"}
{"level":"error","ts":"2026-11-03T15:13:15.421Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"15cd97f1-4469-4d9b-b146-d38b76a3afbe","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-7qq77 not found: certificates.cert-manager.io \"ray-head-cert-long-running-7qq77\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:13:15.472Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"d605ce2d-c696-450f-8181-25db25516aee","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:13:15.472Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"d605ce2d-c696-450f-8181-25db25516aee","name":"long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:13:15.472Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"d605ce2d-c696-450f-8181-25db25516aee","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:13:15.472Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"d605ce2d-c696-450f-8181-25db25516aee"}
{"level":"info","ts":"2026-11-03T15:13:15.472Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"d605ce2d-c696-450f-8181-25db25516aee","RayCluster":"long-running-7qq77","State":""}
{"level":"info","ts":"2026-11-03T15:13:15.472Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"d605ce2d-c696-450f-8181-25db25516aee","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:13:15.503Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"2bb39aef-d45f-4b07-b2c2-b2cd4549242c"}
{"level":"info","ts":"2026-11-03T15:13:15.503Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"2bb39aef-d45f-4b07-b2c2-b2cd4549242c"}
{"level":"info","ts":"2026-11-03T15:13:15.503Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"2bb39aef-d45f-4b07-b2c2-b2cd4549242c","1 head service found":"long-running-7qq77-head-svc"}
{"level":"info","ts":"2026-11-03T15:13:15.503Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"2bb39aef-d45f-4b07-b2c2-b2cd4549242c"}
{"level":"info","ts":"2026-11-03T15:13:15.553Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"2bb39aef-d45f-4b07-b2c2-b2cd4549242c","error":"certificate ray-head-cert-long-running-7qq77 is not ready at current generation 1"}
{"level":"error","ts":"2026-11-03T15:13:15.553Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"2bb39aef-d45f-4b07-b2c2-b2cd4549242c","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-7qq77 is not ready at current generation 1","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:13:15.553Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"2bb39aef-d45f-4b07-b2c2-b2cd4549242c","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-7qq77 is not ready at current generation 1"}
{"level":"info","ts":"2026-11-03T15:13:15.553Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"2bb39aef-d45f-4b07-b2c2-b2cd4549242c","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"long-running-7qq77","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:13:15.553Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"2bb39aef-d45f-4b07-b2c2-b2cd4549242c","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"long-running-7qq77","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:13:15.553Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"2bb39aef-d45f-4b07-b2c2-b2cd4549242c","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"long-running-7qq77","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:13:15.553Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"2bb39aef-d45f-4b07-b2c2-b2cd4549242c"}
{"level":"error","ts":"2026-11-03T15:13:15.553Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"2bb39aef-d45f-4b07-b2c2-b2cd4549242c","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-7qq77 is not ready at current generation 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:13:15.714Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"45d96dfb-a56f-464e-9bc1-ad15f9726671"}
{"level":"info","ts":"2026-11-03T15:13:15.714Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"45d96dfb-a56f-464e-9bc1-ad15f9726671"}
{"level":"info","ts":"2026-11-03T15:13:15.714Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"45d96dfb-a56f-464e-9bc1-ad15f9726671","1 head service found":"long-running-7qq77-head-svc"}
{"level":"info","ts":"2026-11-03T15:13:15.714Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"45d96dfb-a56f-464e-9bc1-ad15f9726671"}
{"level":"info","ts":"2026-11-03T15:13:15.744Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"fd0a9853-3157-4aff-81ad-2b6861b9b53f","certificate":"ray-head-cert-long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:13:15.744Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"fd0a9853-3157-4aff-81ad-2b6861b9b53f"}
{"level":"info","ts":"2026-11-03T15:13:15.744Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"e9874c4d-6737-42ee-a954-651a603ff582","cluster":"long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:13:15.764Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"45d96dfb-a56f-464e-9bc1-ad15f9726671","error":"certificate ray-head-cert-long-running-7qq77 is not ready at current generation 1"}
{"level":"error","ts":"2026-11-03T15:13:15.764Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"45d96dfb-a56f-464e-9bc1-ad15f9726671","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-7qq77 is not ready at current generation 1","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:13:15.764Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"45d96dfb-a56f-464e-9bc1-ad15f9726671","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-7qq77 is not ready at current generation 1"}
{"level":"info","ts":"2026-11-03T15:13:15.764Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"45d96dfb-a56f-464e-9bc1-ad15f9726671","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"long-running-7qq77","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:13:15.764Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"45d96dfb-a56f-464e-9bc1-ad15f9726671","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"long-running-7qq77","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:13:15.764Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"45d96dfb-a56f-464e-9bc1-ad15f9726671","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"long-running-7qq77","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:13:15.764Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"45d96dfb-a56f-464e-9bc1-ad15f9726671"}
{"level":"error","ts":"2026-11-03T15:13:15.764Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"45d96dfb-a56f-464e-9bc1-ad15f9726671","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-7qq77 is not ready at current generation 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:13:16.085Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"2fc54305-cd35-40b4-897f-6b23c9f4e041"}
{"level":"info","ts":"2026-11-03T15:13:16.085Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"2fc54305-cd35-40b4-897f-6b23c9f4e041"}
{"level":"info","ts":"2026-11-03T15:13:16.085Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"2fc54305-cd35-40b4-897f-6b23c9f4e041","1 head service found":"long-running-7qq77-head-svc"}
{"level":"info","ts":"2026-11-03T15:13:16.085Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"2fc54305-cd35-40b4-897f-6b23c9f4e041"}
{"level":"info","ts":"2026-11-03T15:13:16.135Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"2fc54305-cd35-40b4-897f-6b23c9f4e041","error":"certificate ray-head-cert-long-running-7qq77 is not ready at current generation 1"}
{"level":"error","ts":"2026-11-03T15:13:16.135Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"2fc54305-cd35-40b4-897f-6b23c9f4e041","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-7qq77 is not ready at current generation 1","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:13:16.135Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"2fc54305-cd35-40b4-897f-6b23c9f4e041","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-7qq77 is not ready at current generation 1"}
{"level":"info","ts":"2026-11-03T15:13:16.135Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"2fc54305-cd35-40b4-897f-6b23c9f4e041","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"long-running-7qq77","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:13:16.135Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"2fc54305-cd35-40b4-897f-6b23c9f4e041","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"long-running-7qq77","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:13:16.135Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"2fc54305-cd35-40b4-897f-6b23c9f4e041","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"long-running-7qq77","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:13:16.136Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"2fc54305-cd35-40b4-897f-6b23c9f4e041"}
{"level":"error","ts":"2026-11-03T15:13:16.136Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"2fc54305-cd35-40b4-897f-6b23c9f4e041","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-7qq77 is not ready at current generation 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:13:16.192Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"e9874c4d-6737-42ee-a954-651a603ff582","certificate":"ray-head-cert-long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:13:16.192Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"e9874c4d-6737-42ee-a954-651a603ff582"}
{"level":"info","ts":"2026-11-03T15:13:16.776Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"b9fbc585-4ed7-44de-a39b-2c547a2f8e0a"}
{"level":"info","ts":"2026-11-03T15:13:16.777Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"b9fbc585-4ed7-44de-a39b-2c547a2f8e0a"}
{"level":"info","ts":"2026-11-03T15:13:16.777Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"b9fbc585-4ed7-44de-a39b-2c547a2f8e0a","1 head service found":"long-running-7qq77-head-svc"}
{"level":"info","ts":"2026-11-03T15:13:16.777Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"b9fbc585-4ed7-44de-a39b-2c547a2f8e0a"}
{"level":"info","ts":"2026-11-03T15:13:16.827Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"b9fbc585-4ed7-44de-a39b-2c547a2f8e0a","error":"certificate ray-head-cert-long-running-7qq77 is not ready at current generation 1"}
{"level":"error","ts":"2026-11-03T15:13:16.827Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"b9fbc585-4ed7-44de-a39b-2c547a2f8e0a","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-7qq77 is not ready at current generation 1","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:13:16.827Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"b9fbc585-4ed7-44de-a39b-2c547a2f8e0a","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-7qq77 is not ready at current generation 1"}
{"level":"info","ts":"2026-11-03T15:13:16.827Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"b9fbc585-4ed7-44de-a39b-2c547a2f8e0a","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"long-running-7qq77","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:13:16.827Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"b9fbc585-4ed7-44de-a39b-2c547a2f8e0a","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"long-running-7qq77","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:13:16.827Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"b9fbc585-4ed7-44de-a39b-2c547a2f8e0a","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"long-running-7qq77","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:13:16.827Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"b9fbc585-4ed7-44de-a39b-2c547a2f8e0a"}
{"level":"error","ts":"2026-11-03T15:13:16.827Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"b9fbc585-4ed7-44de-a39b-2c547a2f8e0a","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-7qq77 is not ready at current generation 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:13:18.109Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"ecae2864-a112-4e9f-8fdd-fc4118afb5d8"}
{"level":"info","ts":"2026-11-03T15:13:18.109Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"ecae2864-a112-4e9f-8fdd-fc4118afb5d8"}
{"level":"info","ts":"2026-11-03T15:13:18.109Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"ecae2864-a112-4e9f-8fdd-fc4118afb5d8","1 head service found":"long-running-7qq77-head-svc"}
{"level":"info","ts":"2026-11-03T15:13:18.109Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"ecae2864-a112-4e9f-8fdd-fc4118afb5d8"}
{"level":"info","ts":"2026-11-03T15:13:18.210Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"ecae2864-a112-4e9f-8fdd-fc4118afb5d8","mode":"IntegratedOAuth","cluster":"long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:13:18.210Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"ecae2864-a112-4e9f-8fdd-fc4118afb5d8","cluster":"long-running-7qq77","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-11-03T15:13:18.210Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"ecae2864-a112-4e9f-8fdd-fc4118afb5d8","cluster":"long-running-7qq77","authType":"IntegratedOAuth","serviceAccount":"long-running-7qq77-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-11-03T15:13:18.210Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"ecae2864-a112-4e9f-8fdd-fc4118afb5d8","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"long-running-7qq77","ray.io/group":"headgroup","ray.io/identifier":"long-running-7qq77-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-11-03T15:13:18.210Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"ecae2864-a112-4e9f-8fdd-fc4118afb5d8","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-11-03T15:13:18.210Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"ecae2864-a112-4e9f-8fdd-fc4118afb5d8","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-11-03T15:13:18.210Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"ecae2864-a112-4e9f-8fdd-fc4118afb5d8","rayNodeType":"head","generatedCmd":"ulimit -n ${RAY_START_ULIMIT_OPEN_FILES:-65536}; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-11-03T15:13:18.210Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"ecae2864-a112-4e9f-8fdd-fc4118afb5d8","enabled":true}
{"level":"info","ts":"2026-11-03T15:13:18.300Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"ecae2864-a112-4e9f-8fdd-fc4118afb5d8","name":"long-running-7qq77-head-7xmdt"}
{"level":"info","ts":"2026-11-03T15:13:18.300Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"ecae2864-a112-4e9f-8fdd-fc4118afb5d8","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:13:18.300Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"ecae2864-a112-4e9f-8fdd-fc4118afb5d8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:13:18.300Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"ecae2864-a112-4e9f-8fdd-fc4118afb5d8","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-11-03T15:13:18.300Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"ecae2864-a112-4e9f-8fdd-fc4118afb5d8","group":"small-group","indices":{}}
{"level":"info","ts":"2026-11-03T15:13:18.300Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"ecae2864-a112-4e9f-8fdd-fc4118afb5d8","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-11-03T15:13:18.400Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"ecae2864-a112-4e9f-8fdd-fc4118afb5d8","creating worker for group":"small-group","index":0,"total":1,"replicaIndex":0}
{"level":"info","ts":"2026-11-03T15:13:18.400Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"ecae2864-a112-4e9f-8fdd-fc4118afb5d8","nodeType":"worker","rayStartParams":{"address":"long-running-7qq77-head-svc.test-ns-5lqjm.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-11-03T15:13:18.400Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"ecae2864-a112-4e9f-8fdd-fc4118afb5d8","rayStartCmd":"ray start  --address=long-running-7qq77-head-svc.test-ns-5lqjm.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-11-03T15:13:18.400Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"ecae2864-a112-4e9f-8fdd-fc4118afb5d8","rayNodeType":"worker","generatedCmd":"ulimit -n ${RAY_START_ULIMIT_OPEN_FILES:-65536}; ray start  --address=long-running-7qq77-head-svc.test-ns-5lqjm.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-11-03T15:13:18.400Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"ecae2864-a112-4e9f-8fdd-fc4118afb5d8","enabled":true}
{"level":"info","ts":"2026-11-03T15:13:18.462Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"ecae2864-a112-4e9f-8fdd-fc4118afb5d8","name":"long-running-7qq77-small-group-worker-6dvrl"}
{"level":"info","ts":"2026-11-03T15:13:18.462Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"ecae2864-a112-4e9f-8fdd-fc4118afb5d8","podName":"long-running-7qq77-head-7xmdt","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:13:18.462Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"ecae2864-a112-4e9f-8fdd-fc4118afb5d8","podName":"long-running-7qq77-head-7xmdt","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:13:18.462Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"ecae2864-a112-4e9f-8fdd-fc4118afb5d8","name":"long-running-7qq77","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:18Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:13:18.473Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"2158c3f5-c92c-4ac5-8cd5-abb2c7724907","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:13:18.473Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"2158c3f5-c92c-4ac5-8cd5-abb2c7724907","name":"long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:13:18.473Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"2158c3f5-c92c-4ac5-8cd5-abb2c7724907","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:13:18.473Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"2158c3f5-c92c-4ac5-8cd5-abb2c7724907"}
{"level":"info","ts":"2026-11-03T15:13:18.473Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"2158c3f5-c92c-4ac5-8cd5-abb2c7724907","RayCluster":"long-running-7qq77","State":""}
{"level":"info","ts":"2026-11-03T15:13:18.473Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"2158c3f5-c92c-4ac5-8cd5-abb2c7724907","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:13:18.522Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"f18d747f-ddf7-457a-a0a4-6f8802b7ed03"}
{"level":"info","ts":"2026-11-03T15:13:18.522Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"f18d747f-ddf7-457a-a0a4-6f8802b7ed03"}
{"level":"info","ts":"2026-11-03T15:13:18.522Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"f18d747f-ddf7-457a-a0a4-6f8802b7ed03","1 head service found":"long-running-7qq77-head-svc"}
{"level":"info","ts":"2026-11-03T15:13:18.522Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"f18d747f-ddf7-457a-a0a4-6f8802b7ed03","Found 1 head Pod":"long-running-7qq77-head-7xmdt","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:13:18.522Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"f18d747f-ddf7-457a-a0a4-6f8802b7ed03","head Pod":"long-running-7qq77-head-7xmdt","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-7qq77-head-7xmdt. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:13:18.522Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"f18d747f-ddf7-457a-a0a4-6f8802b7ed03","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:13:18.522Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"f18d747f-ddf7-457a-a0a4-6f8802b7ed03","worker Pod":"long-running-7qq77-small-group-worker-6dvrl","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-7qq77-small-group-worker-6dvrl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:13:18.522Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"f18d747f-ddf7-457a-a0a4-6f8802b7ed03","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:13:18.522Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"f18d747f-ddf7-457a-a0a4-6f8802b7ed03","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:13:18.522Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"f18d747f-ddf7-457a-a0a4-6f8802b7ed03","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:13:18.522Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"f18d747f-ddf7-457a-a0a4-6f8802b7ed03","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:13:18.523Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"72e1167c-cc19-443d-b5b8-00b853e5a2fc","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:13:18.523Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"72e1167c-cc19-443d-b5b8-00b853e5a2fc","name":"long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:13:18.523Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"72e1167c-cc19-443d-b5b8-00b853e5a2fc","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:13:18.523Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"f18d747f-ddf7-457a-a0a4-6f8802b7ed03","podName":"long-running-7qq77-head-7xmdt","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:13:18.523Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"f18d747f-ddf7-457a-a0a4-6f8802b7ed03","podName":"long-running-7qq77-head-7xmdt","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:13:18.523Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"f18d747f-ddf7-457a-a0a4-6f8802b7ed03","name":"long-running-7qq77","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:18Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:13:18.523Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"72e1167c-cc19-443d-b5b8-00b853e5a2fc"}
{"level":"info","ts":"2026-11-03T15:13:18.523Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"72e1167c-cc19-443d-b5b8-00b853e5a2fc","RayCluster":"long-running-7qq77","State":""}
{"level":"info","ts":"2026-11-03T15:13:18.523Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"72e1167c-cc19-443d-b5b8-00b853e5a2fc","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:18Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:13:18.523Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"72e1167c-cc19-443d-b5b8-00b853e5a2fc","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:13:18.522Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"6f67002c-a66c-4a46-bf03-dff4707dd5b3","cluster":"long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:13:18.577Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"f18d747f-ddf7-457a-a0a4-6f8802b7ed03","name":"long-running-7qq77","error":"Operation cannot be fulfilled on rayclusters.ray.io \"long-running-7qq77\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-5lqjm","name":"long-running-7qq77"}}
{"level":"info","ts":"2026-11-03T15:13:18.577Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"f18d747f-ddf7-457a-a0a4-6f8802b7ed03"}
{"level":"error","ts":"2026-11-03T15:13:18.577Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"f18d747f-ddf7-457a-a0a4-6f8802b7ed03","error":"Operation cannot be fulfilled on rayclusters.ray.io \"long-running-7qq77\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:13:18.577Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"40ae844c-bd72-42d9-9e49-b946690a1a2c"}
{"level":"info","ts":"2026-11-03T15:13:18.577Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"40ae844c-bd72-42d9-9e49-b946690a1a2c"}
{"level":"info","ts":"2026-11-03T15:13:18.577Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"40ae844c-bd72-42d9-9e49-b946690a1a2c","1 head service found":"long-running-7qq77-head-svc"}
{"level":"info","ts":"2026-11-03T15:13:18.577Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"40ae844c-bd72-42d9-9e49-b946690a1a2c","Found 1 head Pod":"long-running-7qq77-head-7xmdt","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:13:18.577Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"40ae844c-bd72-42d9-9e49-b946690a1a2c","head Pod":"long-running-7qq77-head-7xmdt","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-7qq77-head-7xmdt. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:13:18.577Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"40ae844c-bd72-42d9-9e49-b946690a1a2c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:13:18.577Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"40ae844c-bd72-42d9-9e49-b946690a1a2c","worker Pod":"long-running-7qq77-small-group-worker-6dvrl","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-7qq77-small-group-worker-6dvrl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:13:18.577Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"40ae844c-bd72-42d9-9e49-b946690a1a2c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:13:18.577Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"40ae844c-bd72-42d9-9e49-b946690a1a2c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:13:18.577Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"40ae844c-bd72-42d9-9e49-b946690a1a2c","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:13:18.577Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"40ae844c-bd72-42d9-9e49-b946690a1a2c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:13:18.577Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"40ae844c-bd72-42d9-9e49-b946690a1a2c","podName":"long-running-7qq77-head-7xmdt","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:13:18.577Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"40ae844c-bd72-42d9-9e49-b946690a1a2c","podName":"long-running-7qq77-head-7xmdt","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:13:18.578Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"40ae844c-bd72-42d9-9e49-b946690a1a2c","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:13:18.578Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"40ae844c-bd72-42d9-9e49-b946690a1a2c","seconds":300}
{"level":"info","ts":"2026-11-03T15:13:18.584Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"c9f061ba-edcc-4895-a749-bf2a4c1a3e6b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:13:18.584Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"c9f061ba-edcc-4895-a749-bf2a4c1a3e6b","name":"long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:13:18.584Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"c9f061ba-edcc-4895-a749-bf2a4c1a3e6b","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:13:18.584Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"c9f061ba-edcc-4895-a749-bf2a4c1a3e6b"}
{"level":"info","ts":"2026-11-03T15:13:18.584Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"c9f061ba-edcc-4895-a749-bf2a4c1a3e6b","RayCluster":"long-running-7qq77","State":""}
{"level":"info","ts":"2026-11-03T15:13:18.584Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"c9f061ba-edcc-4895-a749-bf2a4c1a3e6b","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:18Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:18Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:13:18.585Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"1f9f4c8e-aa3f-4814-9282-f31f70caf6d1"}
{"level":"info","ts":"2026-11-03T15:13:18.585Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"1f9f4c8e-aa3f-4814-9282-f31f70caf6d1"}
{"level":"info","ts":"2026-11-03T15:13:18.585Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"1f9f4c8e-aa3f-4814-9282-f31f70caf6d1","1 head service found":"long-running-7qq77-head-svc"}
{"level":"info","ts":"2026-11-03T15:13:18.585Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"1f9f4c8e-aa3f-4814-9282-f31f70caf6d1","Found 1 head Pod":"long-running-7qq77-head-7xmdt","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:13:18.585Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"1f9f4c8e-aa3f-4814-9282-f31f70caf6d1","head Pod":"long-running-7qq77-head-7xmdt","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-7qq77-head-7xmdt. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:13:18.585Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"1f9f4c8e-aa3f-4814-9282-f31f70caf6d1","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:13:18.585Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"1f9f4c8e-aa3f-4814-9282-f31f70caf6d1","worker Pod":"long-running-7qq77-small-group-worker-6dvrl","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-7qq77-small-group-worker-6dvrl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:13:18.585Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"1f9f4c8e-aa3f-4814-9282-f31f70caf6d1","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:13:18.585Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"1f9f4c8e-aa3f-4814-9282-f31f70caf6d1","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:13:18.585Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"1f9f4c8e-aa3f-4814-9282-f31f70caf6d1","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:13:18.585Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"1f9f4c8e-aa3f-4814-9282-f31f70caf6d1","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:13:18.585Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"1f9f4c8e-aa3f-4814-9282-f31f70caf6d1","podName":"long-running-7qq77-small-group-worker-6dvrl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:13:18.585Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"1f9f4c8e-aa3f-4814-9282-f31f70caf6d1","podName":"long-running-7qq77-small-group-worker-6dvrl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:13:18.585Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"1f9f4c8e-aa3f-4814-9282-f31f70caf6d1","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:13:18.585Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"1f9f4c8e-aa3f-4814-9282-f31f70caf6d1","seconds":300}
{"level":"info","ts":"2026-11-03T15:13:19.067Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"6e0ffeec-a27e-4688-b865-54e729b4e29a"}
{"level":"info","ts":"2026-11-03T15:13:19.067Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"6e0ffeec-a27e-4688-b865-54e729b4e29a"}
{"level":"info","ts":"2026-11-03T15:13:19.067Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"6e0ffeec-a27e-4688-b865-54e729b4e29a","1 head service found":"long-running-7qq77-head-svc"}
{"level":"info","ts":"2026-11-03T15:13:19.067Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"6e0ffeec-a27e-4688-b865-54e729b4e29a","Found 1 head Pod":"long-running-7qq77-head-7xmdt","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:13:19.067Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"6e0ffeec-a27e-4688-b865-54e729b4e29a","head Pod":"long-running-7qq77-head-7xmdt","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-7qq77-head-7xmdt. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:13:19.067Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"6e0ffeec-a27e-4688-b865-54e729b4e29a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:13:19.067Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"6e0ffeec-a27e-4688-b865-54e729b4e29a","worker Pod":"long-running-7qq77-small-group-worker-6dvrl","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-7qq77-small-group-worker-6dvrl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:13:19.067Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"6e0ffeec-a27e-4688-b865-54e729b4e29a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:13:19.067Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"6e0ffeec-a27e-4688-b865-54e729b4e29a","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:13:19.067Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"6e0ffeec-a27e-4688-b865-54e729b4e29a","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:13:19.067Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"6e0ffeec-a27e-4688-b865-54e729b4e29a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:13:19.067Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"6e0ffeec-a27e-4688-b865-54e729b4e29a","podName":"long-running-7qq77-head-7xmdt","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:13:19.067Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"6e0ffeec-a27e-4688-b865-54e729b4e29a","podName":"long-running-7qq77-head-7xmdt","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:13:19.067Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"6e0ffeec-a27e-4688-b865-54e729b4e29a","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:13:19.067Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"6e0ffeec-a27e-4688-b865-54e729b4e29a","seconds":300}
{"level":"info","ts":"2026-11-03T15:13:19.212Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"fb9141f5-a38c-4df8-b585-bff746af2fda"}
{"level":"info","ts":"2026-11-03T15:13:19.212Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"fb9141f5-a38c-4df8-b585-bff746af2fda"}
{"level":"info","ts":"2026-11-03T15:13:19.212Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"fb9141f5-a38c-4df8-b585-bff746af2fda","1 head service found":"long-running-7qq77-head-svc"}
{"level":"info","ts":"2026-11-03T15:13:19.212Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"fb9141f5-a38c-4df8-b585-bff746af2fda","Found 1 head Pod":"long-running-7qq77-head-7xmdt","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:13:19.212Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"fb9141f5-a38c-4df8-b585-bff746af2fda","head Pod":"long-running-7qq77-head-7xmdt","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-7qq77-head-7xmdt. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:13:19.212Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"fb9141f5-a38c-4df8-b585-bff746af2fda","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:13:19.212Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"fb9141f5-a38c-4df8-b585-bff746af2fda","worker Pod":"long-running-7qq77-small-group-worker-6dvrl","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-7qq77-small-group-worker-6dvrl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:13:19.212Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"fb9141f5-a38c-4df8-b585-bff746af2fda","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:13:19.212Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"fb9141f5-a38c-4df8-b585-bff746af2fda","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:13:19.212Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"fb9141f5-a38c-4df8-b585-bff746af2fda","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:13:19.212Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"fb9141f5-a38c-4df8-b585-bff746af2fda","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:13:19.212Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"fb9141f5-a38c-4df8-b585-bff746af2fda","podName":"long-running-7qq77-head-7xmdt","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:13:19.212Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"fb9141f5-a38c-4df8-b585-bff746af2fda","podName":"long-running-7qq77-head-7xmdt","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:13:19.212Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"fb9141f5-a38c-4df8-b585-bff746af2fda","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:13:19.212Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"fb9141f5-a38c-4df8-b585-bff746af2fda","seconds":300}
{"level":"info","ts":"2026-11-03T15:13:19.424Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"546e8c14-0f80-4ae0-9837-1c281d9f720b"}
{"level":"info","ts":"2026-11-03T15:13:19.424Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"546e8c14-0f80-4ae0-9837-1c281d9f720b"}
{"level":"info","ts":"2026-11-03T15:13:19.424Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"546e8c14-0f80-4ae0-9837-1c281d9f720b","1 head service found":"long-running-7qq77-head-svc"}
{"level":"info","ts":"2026-11-03T15:13:19.424Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"546e8c14-0f80-4ae0-9837-1c281d9f720b","Found 1 head Pod":"long-running-7qq77-head-7xmdt","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:13:19.424Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"546e8c14-0f80-4ae0-9837-1c281d9f720b","head Pod":"long-running-7qq77-head-7xmdt","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-7qq77-head-7xmdt. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:13:19.424Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"546e8c14-0f80-4ae0-9837-1c281d9f720b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:13:19.424Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"546e8c14-0f80-4ae0-9837-1c281d9f720b","worker Pod":"long-running-7qq77-small-group-worker-6dvrl","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-7qq77-small-group-worker-6dvrl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:13:19.424Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"546e8c14-0f80-4ae0-9837-1c281d9f720b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:13:19.424Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"546e8c14-0f80-4ae0-9837-1c281d9f720b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:13:19.424Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"546e8c14-0f80-4ae0-9837-1c281d9f720b","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:13:19.424Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"546e8c14-0f80-4ae0-9837-1c281d9f720b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:13:19.424Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"546e8c14-0f80-4ae0-9837-1c281d9f720b","podName":"long-running-7qq77-head-7xmdt","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:13:19.424Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"546e8c14-0f80-4ae0-9837-1c281d9f720b","podName":"long-running-7qq77-head-7xmdt","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:13:19.424Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"546e8c14-0f80-4ae0-9837-1c281d9f720b","name":"long-running-7qq77","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:18Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:13:19.485Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"5f131559-85d0-4990-92e2-d1e9238b3e67","cluster":"long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:13:19.485Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"cc93543f-1aaf-4bf3-a24f-b9f3ab75ea04","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:13:19.485Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"cc93543f-1aaf-4bf3-a24f-b9f3ab75ea04","name":"long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:13:19.485Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"cc93543f-1aaf-4bf3-a24f-b9f3ab75ea04","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:13:19.485Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"cc93543f-1aaf-4bf3-a24f-b9f3ab75ea04"}
{"level":"info","ts":"2026-11-03T15:13:19.485Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"cc93543f-1aaf-4bf3-a24f-b9f3ab75ea04","RayCluster":"long-running-7qq77","State":""}
{"level":"info","ts":"2026-11-03T15:13:19.485Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"cc93543f-1aaf-4bf3-a24f-b9f3ab75ea04","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:18Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:13:19.485Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"cc93543f-1aaf-4bf3-a24f-b9f3ab75ea04","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:13:19.546Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"d81db5b3-7055-4be3-810b-d134bb95d2aa","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:13:19.546Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"d81db5b3-7055-4be3-810b-d134bb95d2aa","name":"long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:13:19.546Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"d81db5b3-7055-4be3-810b-d134bb95d2aa","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:13:19.546Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"d81db5b3-7055-4be3-810b-d134bb95d2aa"}
{"level":"info","ts":"2026-11-03T15:13:19.546Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"d81db5b3-7055-4be3-810b-d134bb95d2aa","RayCluster":"long-running-7qq77","State":""}
{"level":"info","ts":"2026-11-03T15:13:19.546Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"d81db5b3-7055-4be3-810b-d134bb95d2aa","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:13:19.791Z","logger":"controllers.RayClusterMTLS","msg":"Updating head certificate","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"5f131559-85d0-4990-92e2-d1e9238b3e67","certificate":"ray-head-cert-long-running-7qq77","ipAddresses":["10.132.0.36","127.0.0.1"]}
{"level":"info","ts":"2026-11-03T15:13:20.202Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"5f131559-85d0-4990-92e2-d1e9238b3e67","certificate":"ray-head-cert-long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:13:20.203Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"5f131559-85d0-4990-92e2-d1e9238b3e67"}
{"level":"info","ts":"2026-11-03T15:13:20.317Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"26671f7a-6b2f-43af-878b-a1a5f4adf9ef"}
{"level":"info","ts":"2026-11-03T15:13:20.317Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"26671f7a-6b2f-43af-878b-a1a5f4adf9ef"}
{"level":"info","ts":"2026-11-03T15:13:20.317Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"26671f7a-6b2f-43af-878b-a1a5f4adf9ef","1 head service found":"long-running-7qq77-head-svc"}
{"level":"info","ts":"2026-11-03T15:13:20.317Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"26671f7a-6b2f-43af-878b-a1a5f4adf9ef","Found 1 head Pod":"long-running-7qq77-head-7xmdt","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:13:20.317Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"26671f7a-6b2f-43af-878b-a1a5f4adf9ef","head Pod":"long-running-7qq77-head-7xmdt","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-7qq77-head-7xmdt. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:13:20.317Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"26671f7a-6b2f-43af-878b-a1a5f4adf9ef","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:13:20.317Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"26671f7a-6b2f-43af-878b-a1a5f4adf9ef","worker Pod":"long-running-7qq77-small-group-worker-6dvrl","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-7qq77-small-group-worker-6dvrl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:13:20.317Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"26671f7a-6b2f-43af-878b-a1a5f4adf9ef","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:13:20.317Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"26671f7a-6b2f-43af-878b-a1a5f4adf9ef","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:13:20.317Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"26671f7a-6b2f-43af-878b-a1a5f4adf9ef","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:13:20.317Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"26671f7a-6b2f-43af-878b-a1a5f4adf9ef","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:13:20.317Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"26671f7a-6b2f-43af-878b-a1a5f4adf9ef","podName":"long-running-7qq77-head-7xmdt","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:13:20.317Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"26671f7a-6b2f-43af-878b-a1a5f4adf9ef","podName":"long-running-7qq77-head-7xmdt","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:13:20.317Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"26671f7a-6b2f-43af-878b-a1a5f4adf9ef","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:13:20.317Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"26671f7a-6b2f-43af-878b-a1a5f4adf9ef","seconds":300}
{"level":"info","ts":"2026-11-03T15:13:22.547Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"94fa36cd-3208-4220-ae74-ad7886e695c2","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:13:22.547Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"94fa36cd-3208-4220-ae74-ad7886e695c2","name":"long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:13:22.547Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"94fa36cd-3208-4220-ae74-ad7886e695c2","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:13:22.547Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"94fa36cd-3208-4220-ae74-ad7886e695c2"}
{"level":"info","ts":"2026-11-03T15:13:22.547Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"94fa36cd-3208-4220-ae74-ad7886e695c2","RayCluster":"long-running-7qq77","State":""}
{"level":"info","ts":"2026-11-03T15:13:22.547Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"94fa36cd-3208-4220-ae74-ad7886e695c2","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:13:25.548Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"71f4f589-f2ab-49af-88b2-92dd19f53925","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:13:25.548Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"71f4f589-f2ab-49af-88b2-92dd19f53925","name":"long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:13:25.548Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"71f4f589-f2ab-49af-88b2-92dd19f53925","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:13:25.548Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"71f4f589-f2ab-49af-88b2-92dd19f53925"}
{"level":"info","ts":"2026-11-03T15:13:25.548Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"71f4f589-f2ab-49af-88b2-92dd19f53925","RayCluster":"long-running-7qq77","State":""}
{"level":"info","ts":"2026-11-03T15:13:25.548Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"71f4f589-f2ab-49af-88b2-92dd19f53925","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:13:28.549Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"90b97ae8-05e4-4465-aa6d-d8efb8f3b840","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:13:28.549Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"90b97ae8-05e4-4465-aa6d-d8efb8f3b840","name":"long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:13:28.549Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"90b97ae8-05e4-4465-aa6d-d8efb8f3b840","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:13:28.549Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"90b97ae8-05e4-4465-aa6d-d8efb8f3b840"}
{"level":"info","ts":"2026-11-03T15:13:28.549Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"90b97ae8-05e4-4465-aa6d-d8efb8f3b840","RayCluster":"long-running-7qq77","State":""}
{"level":"info","ts":"2026-11-03T15:13:28.549Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"90b97ae8-05e4-4465-aa6d-d8efb8f3b840","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:13:31.550Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"1f24cc6d-38a7-4545-9ee3-dddde010ab83","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:13:31.550Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"1f24cc6d-38a7-4545-9ee3-dddde010ab83","name":"long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:13:31.550Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"1f24cc6d-38a7-4545-9ee3-dddde010ab83","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:13:31.550Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"1f24cc6d-38a7-4545-9ee3-dddde010ab83"}
{"level":"info","ts":"2026-11-03T15:13:31.550Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"1f24cc6d-38a7-4545-9ee3-dddde010ab83","RayCluster":"long-running-7qq77","State":""}
{"level":"info","ts":"2026-11-03T15:13:31.550Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"1f24cc6d-38a7-4545-9ee3-dddde010ab83","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:13:34.551Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"8144fea3-d621-4c43-959d-033ec953fefc","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:13:34.551Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"8144fea3-d621-4c43-959d-033ec953fefc","name":"long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:13:34.551Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"8144fea3-d621-4c43-959d-033ec953fefc","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:13:34.552Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"8144fea3-d621-4c43-959d-033ec953fefc"}
{"level":"info","ts":"2026-11-03T15:13:34.552Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"8144fea3-d621-4c43-959d-033ec953fefc","RayCluster":"long-running-7qq77","State":""}
{"level":"info","ts":"2026-11-03T15:13:34.552Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"8144fea3-d621-4c43-959d-033ec953fefc","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:13:37.553Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"6814b49b-8b1d-4fb8-bc6e-f7d93c34325e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:13:37.553Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"6814b49b-8b1d-4fb8-bc6e-f7d93c34325e","name":"long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:13:37.553Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"6814b49b-8b1d-4fb8-bc6e-f7d93c34325e","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:13:37.553Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"6814b49b-8b1d-4fb8-bc6e-f7d93c34325e"}
{"level":"info","ts":"2026-11-03T15:13:37.553Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"6814b49b-8b1d-4fb8-bc6e-f7d93c34325e","RayCluster":"long-running-7qq77","State":""}
{"level":"info","ts":"2026-11-03T15:13:37.553Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"6814b49b-8b1d-4fb8-bc6e-f7d93c34325e","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:13:40.554Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"4b870667-bc0f-47e5-b86e-1ef356bb6b02","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:13:40.554Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"4b870667-bc0f-47e5-b86e-1ef356bb6b02","name":"long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:13:40.554Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"4b870667-bc0f-47e5-b86e-1ef356bb6b02","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:13:40.555Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"4b870667-bc0f-47e5-b86e-1ef356bb6b02"}
{"level":"info","ts":"2026-11-03T15:13:40.555Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"4b870667-bc0f-47e5-b86e-1ef356bb6b02","RayCluster":"long-running-7qq77","State":""}
{"level":"info","ts":"2026-11-03T15:13:40.555Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"4b870667-bc0f-47e5-b86e-1ef356bb6b02","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:13:43.556Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"ef893af4-8417-4c8c-87d4-327efcd1565a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:13:43.556Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"ef893af4-8417-4c8c-87d4-327efcd1565a","name":"long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:13:43.556Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"ef893af4-8417-4c8c-87d4-327efcd1565a","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:13:43.556Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"ef893af4-8417-4c8c-87d4-327efcd1565a"}
{"level":"info","ts":"2026-11-03T15:13:43.556Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"ef893af4-8417-4c8c-87d4-327efcd1565a","RayCluster":"long-running-7qq77","State":""}
{"level":"info","ts":"2026-11-03T15:13:43.556Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"ef893af4-8417-4c8c-87d4-327efcd1565a","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:13:46.557Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"cf2e9c02-d859-4a6d-9ee7-33f93fe5ee95","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:13:46.557Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"cf2e9c02-d859-4a6d-9ee7-33f93fe5ee95","name":"long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:13:46.557Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"cf2e9c02-d859-4a6d-9ee7-33f93fe5ee95","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:13:46.557Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"cf2e9c02-d859-4a6d-9ee7-33f93fe5ee95"}
{"level":"info","ts":"2026-11-03T15:13:46.557Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"cf2e9c02-d859-4a6d-9ee7-33f93fe5ee95","RayCluster":"long-running-7qq77","State":""}
{"level":"info","ts":"2026-11-03T15:13:46.557Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"cf2e9c02-d859-4a6d-9ee7-33f93fe5ee95","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:13:49.558Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"3cf41260-ac27-4eef-a5b8-28d8a72bca80","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:13:49.558Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"3cf41260-ac27-4eef-a5b8-28d8a72bca80","name":"long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:13:49.558Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"3cf41260-ac27-4eef-a5b8-28d8a72bca80","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:13:49.558Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"3cf41260-ac27-4eef-a5b8-28d8a72bca80"}
{"level":"info","ts":"2026-11-03T15:13:49.558Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"3cf41260-ac27-4eef-a5b8-28d8a72bca80","RayCluster":"long-running-7qq77","State":""}
{"level":"info","ts":"2026-11-03T15:13:49.558Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"3cf41260-ac27-4eef-a5b8-28d8a72bca80","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:13:50.203Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"258ed604-f104-4bc2-914e-79ddbc004d65","cluster":"long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:13:50.658Z","logger":"controllers.RayClusterMTLS","msg":"Updating worker certificate","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"258ed604-f104-4bc2-914e-79ddbc004d65","certificate":"ray-worker-cert-long-running-7qq77","ipAddresses":["10.134.0.33","127.0.0.1"]}
{"level":"info","ts":"2026-11-03T15:13:50.970Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"258ed604-f104-4bc2-914e-79ddbc004d65","certificate":"ray-worker-cert-long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:13:50.970Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"258ed604-f104-4bc2-914e-79ddbc004d65"}
{"level":"info","ts":"2026-11-03T15:13:52.559Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"60d0e00f-8831-46ae-80fa-68bb6688f9c2","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:13:52.559Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"60d0e00f-8831-46ae-80fa-68bb6688f9c2","name":"long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:13:52.559Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"60d0e00f-8831-46ae-80fa-68bb6688f9c2","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:13:52.559Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"60d0e00f-8831-46ae-80fa-68bb6688f9c2"}
{"level":"info","ts":"2026-11-03T15:13:52.559Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"60d0e00f-8831-46ae-80fa-68bb6688f9c2","RayCluster":"long-running-7qq77","State":""}
{"level":"info","ts":"2026-11-03T15:13:52.559Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"60d0e00f-8831-46ae-80fa-68bb6688f9c2","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:13:55.560Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"c66ac96e-5edb-4970-837f-b2ac39be009b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:13:55.560Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"c66ac96e-5edb-4970-837f-b2ac39be009b","name":"long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:13:55.560Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"c66ac96e-5edb-4970-837f-b2ac39be009b","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:13:55.560Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"c66ac96e-5edb-4970-837f-b2ac39be009b"}
{"level":"info","ts":"2026-11-03T15:13:55.560Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"c66ac96e-5edb-4970-837f-b2ac39be009b","RayCluster":"long-running-7qq77","State":""}
{"level":"info","ts":"2026-11-03T15:13:55.560Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"c66ac96e-5edb-4970-837f-b2ac39be009b","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:13:58.561Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"ca1e3097-3f8b-40b5-bf19-ea0030bae4d7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:13:58.561Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"ca1e3097-3f8b-40b5-bf19-ea0030bae4d7","name":"long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:13:58.561Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"ca1e3097-3f8b-40b5-bf19-ea0030bae4d7","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:13:58.562Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"ca1e3097-3f8b-40b5-bf19-ea0030bae4d7"}
{"level":"info","ts":"2026-11-03T15:13:58.562Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"ca1e3097-3f8b-40b5-bf19-ea0030bae4d7","RayCluster":"long-running-7qq77","State":""}
{"level":"info","ts":"2026-11-03T15:13:58.562Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"ca1e3097-3f8b-40b5-bf19-ea0030bae4d7","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:14:01.562Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"2fd104dc-0b00-4c00-8f7d-4339a3065c29","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:14:01.562Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"2fd104dc-0b00-4c00-8f7d-4339a3065c29","name":"long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:14:01.562Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"2fd104dc-0b00-4c00-8f7d-4339a3065c29","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:14:01.563Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"2fd104dc-0b00-4c00-8f7d-4339a3065c29"}
{"level":"info","ts":"2026-11-03T15:14:01.563Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"2fd104dc-0b00-4c00-8f7d-4339a3065c29","RayCluster":"long-running-7qq77","State":""}
{"level":"info","ts":"2026-11-03T15:14:01.563Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"2fd104dc-0b00-4c00-8f7d-4339a3065c29","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:14:04.563Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"8b69bbca-7ccc-43fe-881d-964a025a1dfc","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:14:04.563Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"8b69bbca-7ccc-43fe-881d-964a025a1dfc","name":"long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:14:04.563Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"8b69bbca-7ccc-43fe-881d-964a025a1dfc","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:14:04.563Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"8b69bbca-7ccc-43fe-881d-964a025a1dfc"}
{"level":"info","ts":"2026-11-03T15:14:04.563Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"8b69bbca-7ccc-43fe-881d-964a025a1dfc","RayCluster":"long-running-7qq77","State":""}
{"level":"info","ts":"2026-11-03T15:14:04.563Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"8b69bbca-7ccc-43fe-881d-964a025a1dfc","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:14:07.564Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"22ac0a24-dc47-461c-ba26-c5bffcb5f61b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:14:07.564Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"22ac0a24-dc47-461c-ba26-c5bffcb5f61b","name":"long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:14:07.564Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"22ac0a24-dc47-461c-ba26-c5bffcb5f61b","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:14:07.564Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"22ac0a24-dc47-461c-ba26-c5bffcb5f61b"}
{"level":"info","ts":"2026-11-03T15:14:07.564Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"22ac0a24-dc47-461c-ba26-c5bffcb5f61b","RayCluster":"long-running-7qq77","State":""}
{"level":"info","ts":"2026-11-03T15:14:07.564Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"22ac0a24-dc47-461c-ba26-c5bffcb5f61b","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:14:10.565Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"17faaba4-3542-4ca5-bb5f-2a917e19cb28","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:14:10.565Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"17faaba4-3542-4ca5-bb5f-2a917e19cb28","name":"long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:14:10.565Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"17faaba4-3542-4ca5-bb5f-2a917e19cb28","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:14:10.566Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"17faaba4-3542-4ca5-bb5f-2a917e19cb28"}
{"level":"info","ts":"2026-11-03T15:14:10.566Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"17faaba4-3542-4ca5-bb5f-2a917e19cb28","RayCluster":"long-running-7qq77","State":""}
{"level":"info","ts":"2026-11-03T15:14:10.566Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"17faaba4-3542-4ca5-bb5f-2a917e19cb28","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:14:13.567Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"1988eca6-e7ef-40cc-a899-05d4ba1c37af","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:14:13.567Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"1988eca6-e7ef-40cc-a899-05d4ba1c37af","name":"long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:14:13.567Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"1988eca6-e7ef-40cc-a899-05d4ba1c37af","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:14:13.567Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"1988eca6-e7ef-40cc-a899-05d4ba1c37af"}
{"level":"info","ts":"2026-11-03T15:14:13.567Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"1988eca6-e7ef-40cc-a899-05d4ba1c37af","RayCluster":"long-running-7qq77","State":""}
{"level":"info","ts":"2026-11-03T15:14:13.567Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"1988eca6-e7ef-40cc-a899-05d4ba1c37af","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:14:16.568Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"4e4ee29a-be22-4305-b090-e50fca7282f1","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:14:16.568Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"4e4ee29a-be22-4305-b090-e50fca7282f1","name":"long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:14:16.568Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"4e4ee29a-be22-4305-b090-e50fca7282f1","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:14:16.569Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"4e4ee29a-be22-4305-b090-e50fca7282f1"}
{"level":"info","ts":"2026-11-03T15:14:16.569Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"4e4ee29a-be22-4305-b090-e50fca7282f1","RayCluster":"long-running-7qq77","State":""}
{"level":"info","ts":"2026-11-03T15:14:16.569Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"4e4ee29a-be22-4305-b090-e50fca7282f1","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:14:19.569Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"5ecad457-7f18-4de5-a38f-cf48fadc2e70","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:14:19.569Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"5ecad457-7f18-4de5-a38f-cf48fadc2e70","name":"long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:14:19.569Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"5ecad457-7f18-4de5-a38f-cf48fadc2e70","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:14:19.570Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"5ecad457-7f18-4de5-a38f-cf48fadc2e70"}
{"level":"info","ts":"2026-11-03T15:14:19.570Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"5ecad457-7f18-4de5-a38f-cf48fadc2e70","RayCluster":"long-running-7qq77","State":""}
{"level":"info","ts":"2026-11-03T15:14:19.570Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"5ecad457-7f18-4de5-a38f-cf48fadc2e70","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:14:20.971Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"7bc5d8e5-99c8-40f1-921a-e3242a6736d9","cluster":"long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:14:22.570Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"25eb2ac4-39fb-47be-a63d-0f54564e1075","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:14:22.570Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"25eb2ac4-39fb-47be-a63d-0f54564e1075","name":"long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:14:22.570Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"25eb2ac4-39fb-47be-a63d-0f54564e1075","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:14:22.570Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"25eb2ac4-39fb-47be-a63d-0f54564e1075"}
{"level":"info","ts":"2026-11-03T15:14:22.570Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"25eb2ac4-39fb-47be-a63d-0f54564e1075","RayCluster":"long-running-7qq77","State":""}
{"level":"info","ts":"2026-11-03T15:14:22.570Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"25eb2ac4-39fb-47be-a63d-0f54564e1075","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:14:25.571Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"ab12082d-f714-439a-ba25-61d56fe82937","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:14:25.571Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"ab12082d-f714-439a-ba25-61d56fe82937","name":"long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:14:25.571Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"ab12082d-f714-439a-ba25-61d56fe82937","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:14:25.572Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"ab12082d-f714-439a-ba25-61d56fe82937"}
{"level":"info","ts":"2026-11-03T15:14:25.572Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"ab12082d-f714-439a-ba25-61d56fe82937","RayCluster":"long-running-7qq77","State":""}
{"level":"info","ts":"2026-11-03T15:14:25.572Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"ab12082d-f714-439a-ba25-61d56fe82937","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:14:28.572Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"381a98db-091a-4f1f-a138-08eee6bdddc3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:14:28.572Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"381a98db-091a-4f1f-a138-08eee6bdddc3","name":"long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:14:28.572Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"381a98db-091a-4f1f-a138-08eee6bdddc3","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:14:28.573Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"381a98db-091a-4f1f-a138-08eee6bdddc3"}
{"level":"info","ts":"2026-11-03T15:14:28.573Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"381a98db-091a-4f1f-a138-08eee6bdddc3","RayCluster":"long-running-7qq77","State":""}
{"level":"info","ts":"2026-11-03T15:14:28.573Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"381a98db-091a-4f1f-a138-08eee6bdddc3","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:14:29.615Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"fbfe5150-47f5-4057-b2c3-1c6bd7ea7f33"}
{"level":"info","ts":"2026-11-03T15:14:29.615Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"fbfe5150-47f5-4057-b2c3-1c6bd7ea7f33"}
{"level":"info","ts":"2026-11-03T15:14:29.615Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"fbfe5150-47f5-4057-b2c3-1c6bd7ea7f33","1 head service found":"long-running-7qq77-head-svc"}
{"level":"info","ts":"2026-11-03T15:14:29.615Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"fbfe5150-47f5-4057-b2c3-1c6bd7ea7f33","Found 1 head Pod":"long-running-7qq77-head-7xmdt","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:14:29.616Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"fbfe5150-47f5-4057-b2c3-1c6bd7ea7f33","head Pod":"long-running-7qq77-head-7xmdt","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-7qq77-head-7xmdt. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:14:29.616Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"fbfe5150-47f5-4057-b2c3-1c6bd7ea7f33","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:14:29.616Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"fbfe5150-47f5-4057-b2c3-1c6bd7ea7f33","worker Pod":"long-running-7qq77-small-group-worker-6dvrl","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-7qq77-small-group-worker-6dvrl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:14:29.616Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"fbfe5150-47f5-4057-b2c3-1c6bd7ea7f33","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:14:29.616Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"fbfe5150-47f5-4057-b2c3-1c6bd7ea7f33","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:14:29.616Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"fbfe5150-47f5-4057-b2c3-1c6bd7ea7f33","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:14:29.616Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"fbfe5150-47f5-4057-b2c3-1c6bd7ea7f33","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:14:29.616Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"fbfe5150-47f5-4057-b2c3-1c6bd7ea7f33","podName":"long-running-7qq77-head-7xmdt","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:14:29.616Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"fbfe5150-47f5-4057-b2c3-1c6bd7ea7f33","podName":"long-running-7qq77-head-7xmdt","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:14:29.616Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"fbfe5150-47f5-4057-b2c3-1c6bd7ea7f33","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:14:29.616Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"fbfe5150-47f5-4057-b2c3-1c6bd7ea7f33","seconds":300}
{"level":"info","ts":"2026-11-03T15:14:30.622Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"a22ef79a-a453-41c4-88c8-4ee7ba23321d"}
{"level":"info","ts":"2026-11-03T15:14:30.622Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"a22ef79a-a453-41c4-88c8-4ee7ba23321d"}
{"level":"info","ts":"2026-11-03T15:14:30.622Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"a22ef79a-a453-41c4-88c8-4ee7ba23321d","1 head service found":"long-running-7qq77-head-svc"}
{"level":"info","ts":"2026-11-03T15:14:30.622Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"a22ef79a-a453-41c4-88c8-4ee7ba23321d","Found 1 head Pod":"long-running-7qq77-head-7xmdt","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:14:30.622Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"a22ef79a-a453-41c4-88c8-4ee7ba23321d","head Pod":"long-running-7qq77-head-7xmdt","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-7qq77-head-7xmdt. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:14:30.622Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"a22ef79a-a453-41c4-88c8-4ee7ba23321d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:14:30.622Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"a22ef79a-a453-41c4-88c8-4ee7ba23321d","worker Pod":"long-running-7qq77-small-group-worker-6dvrl","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-7qq77-small-group-worker-6dvrl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:14:30.622Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"a22ef79a-a453-41c4-88c8-4ee7ba23321d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:14:30.622Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"a22ef79a-a453-41c4-88c8-4ee7ba23321d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:14:30.622Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"a22ef79a-a453-41c4-88c8-4ee7ba23321d","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:14:30.622Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"a22ef79a-a453-41c4-88c8-4ee7ba23321d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:14:30.622Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"a22ef79a-a453-41c4-88c8-4ee7ba23321d","podName":"long-running-7qq77-small-group-worker-6dvrl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:14:30.622Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"a22ef79a-a453-41c4-88c8-4ee7ba23321d","podName":"long-running-7qq77-small-group-worker-6dvrl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:14:30.622Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"a22ef79a-a453-41c4-88c8-4ee7ba23321d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:14:30.622Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"a22ef79a-a453-41c4-88c8-4ee7ba23321d","seconds":300}
{"level":"info","ts":"2026-11-03T15:14:31.573Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"902b03f4-7618-4491-a95c-3d4c5418abd7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:14:31.573Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"902b03f4-7618-4491-a95c-3d4c5418abd7","name":"long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:14:31.573Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"902b03f4-7618-4491-a95c-3d4c5418abd7","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:14:31.573Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"902b03f4-7618-4491-a95c-3d4c5418abd7"}
{"level":"info","ts":"2026-11-03T15:14:31.573Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"902b03f4-7618-4491-a95c-3d4c5418abd7","RayCluster":"long-running-7qq77","State":""}
{"level":"info","ts":"2026-11-03T15:14:31.573Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"902b03f4-7618-4491-a95c-3d4c5418abd7","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:14:34.574Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"ae8d88ac-9ea8-4a40-9118-b065569f9f38","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:14:34.574Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"ae8d88ac-9ea8-4a40-9118-b065569f9f38","name":"long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:14:34.574Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"ae8d88ac-9ea8-4a40-9118-b065569f9f38","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:14:34.575Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"ae8d88ac-9ea8-4a40-9118-b065569f9f38"}
{"level":"info","ts":"2026-11-03T15:14:34.575Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"ae8d88ac-9ea8-4a40-9118-b065569f9f38","RayCluster":"long-running-7qq77","State":""}
{"level":"info","ts":"2026-11-03T15:14:34.575Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"ae8d88ac-9ea8-4a40-9118-b065569f9f38","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:14:34.819Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"371273dd-3ec4-445d-a9ee-56edd7230558"}
{"level":"info","ts":"2026-11-03T15:14:34.819Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"371273dd-3ec4-445d-a9ee-56edd7230558"}
{"level":"info","ts":"2026-11-03T15:14:34.820Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"371273dd-3ec4-445d-a9ee-56edd7230558","1 head service found":"long-running-7qq77-head-svc"}
{"level":"info","ts":"2026-11-03T15:14:34.820Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"371273dd-3ec4-445d-a9ee-56edd7230558","Found 1 head Pod":"long-running-7qq77-head-7xmdt","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:14:34.820Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"371273dd-3ec4-445d-a9ee-56edd7230558","head Pod":"long-running-7qq77-head-7xmdt","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-7qq77-head-7xmdt. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:14:34.820Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"371273dd-3ec4-445d-a9ee-56edd7230558","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:14:34.820Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"371273dd-3ec4-445d-a9ee-56edd7230558","worker Pod":"long-running-7qq77-small-group-worker-6dvrl","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-7qq77-small-group-worker-6dvrl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:14:34.820Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"371273dd-3ec4-445d-a9ee-56edd7230558","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:14:34.820Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"371273dd-3ec4-445d-a9ee-56edd7230558","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:14:34.820Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"371273dd-3ec4-445d-a9ee-56edd7230558","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:14:34.820Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"371273dd-3ec4-445d-a9ee-56edd7230558","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:14:34.820Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"371273dd-3ec4-445d-a9ee-56edd7230558","podName":"long-running-7qq77-head-7xmdt","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:14:34.820Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"371273dd-3ec4-445d-a9ee-56edd7230558","podName":"long-running-7qq77-head-7xmdt","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:14:34.820Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"371273dd-3ec4-445d-a9ee-56edd7230558","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:14:34.820Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"371273dd-3ec4-445d-a9ee-56edd7230558","seconds":300}
{"level":"info","ts":"2026-11-03T15:14:35.827Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"77c5d0b2-da02-4bf1-a112-6bc350383dc3"}
{"level":"info","ts":"2026-11-03T15:14:35.827Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"77c5d0b2-da02-4bf1-a112-6bc350383dc3"}
{"level":"info","ts":"2026-11-03T15:14:35.827Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"77c5d0b2-da02-4bf1-a112-6bc350383dc3","1 head service found":"long-running-7qq77-head-svc"}
{"level":"info","ts":"2026-11-03T15:14:35.827Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"77c5d0b2-da02-4bf1-a112-6bc350383dc3","Found 1 head Pod":"long-running-7qq77-head-7xmdt","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:14:35.827Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"77c5d0b2-da02-4bf1-a112-6bc350383dc3","head Pod":"long-running-7qq77-head-7xmdt","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-7qq77-head-7xmdt. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:14:35.827Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"77c5d0b2-da02-4bf1-a112-6bc350383dc3","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:14:35.827Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"77c5d0b2-da02-4bf1-a112-6bc350383dc3","worker Pod":"long-running-7qq77-small-group-worker-6dvrl","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-7qq77-small-group-worker-6dvrl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:14:35.827Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"77c5d0b2-da02-4bf1-a112-6bc350383dc3","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:14:35.827Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"77c5d0b2-da02-4bf1-a112-6bc350383dc3","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:14:35.827Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"77c5d0b2-da02-4bf1-a112-6bc350383dc3","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:14:35.827Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"77c5d0b2-da02-4bf1-a112-6bc350383dc3","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:14:35.827Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"77c5d0b2-da02-4bf1-a112-6bc350383dc3","podName":"long-running-7qq77-head-7xmdt","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:13:18Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:14:35.827Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"77c5d0b2-da02-4bf1-a112-6bc350383dc3","podName":"long-running-7qq77-head-7xmdt","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:13:18Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:14:35.827Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"77c5d0b2-da02-4bf1-a112-6bc350383dc3","name":"long-running-7qq77","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:14:35.888Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"a8fab933-a7be-45b0-bd12-ffb15bc44ecf","cluster":"long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:14:35.888Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"16a525b3-c1c0-4429-90ad-3d8487f46ce3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:14:35.888Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"16a525b3-c1c0-4429-90ad-3d8487f46ce3","name":"long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:14:35.888Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"16a525b3-c1c0-4429-90ad-3d8487f46ce3","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:14:35.889Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"16a525b3-c1c0-4429-90ad-3d8487f46ce3"}
{"level":"info","ts":"2026-11-03T15:14:35.889Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"16a525b3-c1c0-4429-90ad-3d8487f46ce3","RayCluster":"long-running-7qq77","State":""}
{"level":"info","ts":"2026-11-03T15:14:35.889Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"16a525b3-c1c0-4429-90ad-3d8487f46ce3","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:13:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:14:35.889Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"16a525b3-c1c0-4429-90ad-3d8487f46ce3","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:14:35.949Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"8b78bc11-fc3f-42df-aa06-95c7bbe06280","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:14:35.949Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"8b78bc11-fc3f-42df-aa06-95c7bbe06280","name":"long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:14:35.949Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"8b78bc11-fc3f-42df-aa06-95c7bbe06280","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:14:35.950Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"8b78bc11-fc3f-42df-aa06-95c7bbe06280"}
{"level":"info","ts":"2026-11-03T15:14:35.950Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"8b78bc11-fc3f-42df-aa06-95c7bbe06280","RayCluster":"long-running-7qq77","State":""}
{"level":"info","ts":"2026-11-03T15:14:35.950Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"8b78bc11-fc3f-42df-aa06-95c7bbe06280","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:14:37.888Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"3e69d27e-400d-4281-acf0-a87f760a7443"}
{"level":"info","ts":"2026-11-03T15:14:37.888Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"3e69d27e-400d-4281-acf0-a87f760a7443"}
{"level":"info","ts":"2026-11-03T15:14:37.888Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"3e69d27e-400d-4281-acf0-a87f760a7443","1 head service found":"long-running-7qq77-head-svc"}
{"level":"info","ts":"2026-11-03T15:14:37.888Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"3e69d27e-400d-4281-acf0-a87f760a7443","Found 1 head Pod":"long-running-7qq77-head-7xmdt","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:14:37.888Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"3e69d27e-400d-4281-acf0-a87f760a7443","head Pod":"long-running-7qq77-head-7xmdt","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-7qq77-head-7xmdt. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:14:37.888Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"3e69d27e-400d-4281-acf0-a87f760a7443","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:14:37.888Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"3e69d27e-400d-4281-acf0-a87f760a7443","worker Pod":"long-running-7qq77-small-group-worker-6dvrl","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-7qq77-small-group-worker-6dvrl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:14:37.888Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"3e69d27e-400d-4281-acf0-a87f760a7443","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:14:37.888Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"3e69d27e-400d-4281-acf0-a87f760a7443","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:14:37.889Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"3e69d27e-400d-4281-acf0-a87f760a7443","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:14:37.889Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"3e69d27e-400d-4281-acf0-a87f760a7443","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:14:37.889Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"3e69d27e-400d-4281-acf0-a87f760a7443","podName":"long-running-7qq77-head-7xmdt","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:13:18Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:14:37.889Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"3e69d27e-400d-4281-acf0-a87f760a7443","podName":"long-running-7qq77-head-7xmdt","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:13:18Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:14:37.889Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"3e69d27e-400d-4281-acf0-a87f760a7443","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:14:37.889Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"3e69d27e-400d-4281-acf0-a87f760a7443","seconds":300}
{"level":"info","ts":"2026-11-03T15:14:38.950Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"7e36b994-be9a-4dc9-ad12-3cc2d2fbca84","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:14:38.950Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"7e36b994-be9a-4dc9-ad12-3cc2d2fbca84","name":"long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:14:38.950Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"7e36b994-be9a-4dc9-ad12-3cc2d2fbca84","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:14:38.950Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"7e36b994-be9a-4dc9-ad12-3cc2d2fbca84"}
{"level":"info","ts":"2026-11-03T15:14:38.950Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"7e36b994-be9a-4dc9-ad12-3cc2d2fbca84","RayCluster":"long-running-7qq77","State":""}
{"level":"info","ts":"2026-11-03T15:14:38.950Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"7e36b994-be9a-4dc9-ad12-3cc2d2fbca84","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:14:39.664Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"b301b12f-a03e-46f3-953b-7b8a08ba93fd"}
{"level":"info","ts":"2026-11-03T15:14:39.664Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"b301b12f-a03e-46f3-953b-7b8a08ba93fd"}
{"level":"info","ts":"2026-11-03T15:14:39.664Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"b301b12f-a03e-46f3-953b-7b8a08ba93fd","1 head service found":"long-running-7qq77-head-svc"}
{"level":"info","ts":"2026-11-03T15:14:39.664Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"b301b12f-a03e-46f3-953b-7b8a08ba93fd","Found 1 head Pod":"long-running-7qq77-head-7xmdt","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:14:39.664Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"b301b12f-a03e-46f3-953b-7b8a08ba93fd","head Pod":"long-running-7qq77-head-7xmdt","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-7qq77-head-7xmdt. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:14:39.664Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"b301b12f-a03e-46f3-953b-7b8a08ba93fd","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:14:39.664Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"b301b12f-a03e-46f3-953b-7b8a08ba93fd","worker Pod":"long-running-7qq77-small-group-worker-6dvrl","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-7qq77-small-group-worker-6dvrl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:14:39.664Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"b301b12f-a03e-46f3-953b-7b8a08ba93fd","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:14:39.664Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"b301b12f-a03e-46f3-953b-7b8a08ba93fd","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:14:39.664Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"b301b12f-a03e-46f3-953b-7b8a08ba93fd","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:14:39.664Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"b301b12f-a03e-46f3-953b-7b8a08ba93fd","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:14:39.664Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"b301b12f-a03e-46f3-953b-7b8a08ba93fd","podName":"long-running-7qq77-head-7xmdt","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:13:18Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:14:39.664Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"b301b12f-a03e-46f3-953b-7b8a08ba93fd","podName":"long-running-7qq77-head-7xmdt","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:13:18Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:14:39.664Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"b301b12f-a03e-46f3-953b-7b8a08ba93fd","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:14:39.664Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"b301b12f-a03e-46f3-953b-7b8a08ba93fd","seconds":300}
{"level":"info","ts":"2026-11-03T15:14:40.719Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"bf152853-0c41-48ec-8b1a-5948981982c1"}
{"level":"info","ts":"2026-11-03T15:14:40.719Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"bf152853-0c41-48ec-8b1a-5948981982c1"}
{"level":"info","ts":"2026-11-03T15:14:40.719Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"bf152853-0c41-48ec-8b1a-5948981982c1","1 head service found":"long-running-7qq77-head-svc"}
{"level":"info","ts":"2026-11-03T15:14:40.719Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"bf152853-0c41-48ec-8b1a-5948981982c1","Found 1 head Pod":"long-running-7qq77-head-7xmdt","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:14:40.719Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"bf152853-0c41-48ec-8b1a-5948981982c1","head Pod":"long-running-7qq77-head-7xmdt","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-7qq77-head-7xmdt. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:14:40.719Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"bf152853-0c41-48ec-8b1a-5948981982c1","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:14:40.719Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"bf152853-0c41-48ec-8b1a-5948981982c1","worker Pod":"long-running-7qq77-small-group-worker-6dvrl","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-7qq77-small-group-worker-6dvrl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:14:40.719Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"bf152853-0c41-48ec-8b1a-5948981982c1","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:14:40.719Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"bf152853-0c41-48ec-8b1a-5948981982c1","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:14:40.719Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"bf152853-0c41-48ec-8b1a-5948981982c1","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:14:40.719Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"bf152853-0c41-48ec-8b1a-5948981982c1","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:14:40.720Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"bf152853-0c41-48ec-8b1a-5948981982c1","podName":"long-running-7qq77-head-7xmdt","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:13:18Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:14:40.720Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"bf152853-0c41-48ec-8b1a-5948981982c1","podName":"long-running-7qq77-head-7xmdt","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:13:18Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:14:40.720Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"bf152853-0c41-48ec-8b1a-5948981982c1","name":"long-running-7qq77","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:40Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:14:40.780Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"fbac609f-a93b-4d88-8275-c478d6a8cd5b","cluster":"long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:14:40.780Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"4c46cb17-6f30-42a4-84a0-63305706ada8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:14:40.780Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"4c46cb17-6f30-42a4-84a0-63305706ada8","name":"long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:14:40.780Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"4c46cb17-6f30-42a4-84a0-63305706ada8","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:14:40.781Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"4c46cb17-6f30-42a4-84a0-63305706ada8"}
{"level":"info","ts":"2026-11-03T15:14:40.781Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"4c46cb17-6f30-42a4-84a0-63305706ada8","RayCluster":"long-running-7qq77","State":""}
{"level":"info","ts":"2026-11-03T15:14:40.781Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"4c46cb17-6f30-42a4-84a0-63305706ada8","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:40Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:14:40.781Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"4c46cb17-6f30-42a4-84a0-63305706ada8","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:14:40.841Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"e3cdfdda-b4a4-4906-8d5c-fc01515ede0b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:14:40.841Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"e3cdfdda-b4a4-4906-8d5c-fc01515ede0b","name":"long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:14:40.841Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"e3cdfdda-b4a4-4906-8d5c-fc01515ede0b","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:14:40.841Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"e3cdfdda-b4a4-4906-8d5c-fc01515ede0b"}
{"level":"info","ts":"2026-11-03T15:14:40.841Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"e3cdfdda-b4a4-4906-8d5c-fc01515ede0b","RayCluster":"long-running-7qq77","State":""}
{"level":"info","ts":"2026-11-03T15:14:40.841Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"e3cdfdda-b4a4-4906-8d5c-fc01515ede0b","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:40Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:40Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:14:42.780Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"be111511-d320-425e-b9d8-63b03b10f264"}
{"level":"info","ts":"2026-11-03T15:14:42.780Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"be111511-d320-425e-b9d8-63b03b10f264"}
{"level":"info","ts":"2026-11-03T15:14:42.780Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"be111511-d320-425e-b9d8-63b03b10f264","1 head service found":"long-running-7qq77-head-svc"}
{"level":"info","ts":"2026-11-03T15:14:42.781Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"be111511-d320-425e-b9d8-63b03b10f264","Found 1 head Pod":"long-running-7qq77-head-7xmdt","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:14:42.781Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"be111511-d320-425e-b9d8-63b03b10f264","head Pod":"long-running-7qq77-head-7xmdt","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-7qq77-head-7xmdt. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:14:42.781Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"be111511-d320-425e-b9d8-63b03b10f264","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:14:42.781Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"be111511-d320-425e-b9d8-63b03b10f264","worker Pod":"long-running-7qq77-small-group-worker-6dvrl","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-7qq77-small-group-worker-6dvrl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:14:42.781Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"be111511-d320-425e-b9d8-63b03b10f264","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:14:42.781Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"be111511-d320-425e-b9d8-63b03b10f264","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:14:42.781Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"be111511-d320-425e-b9d8-63b03b10f264","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:14:42.781Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"be111511-d320-425e-b9d8-63b03b10f264","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:14:42.781Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"be111511-d320-425e-b9d8-63b03b10f264","podName":"long-running-7qq77-head-7xmdt","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:13:18Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:14:42.781Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"be111511-d320-425e-b9d8-63b03b10f264","podName":"long-running-7qq77-head-7xmdt","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:13:18Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:14:42.781Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"be111511-d320-425e-b9d8-63b03b10f264","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:14:42.781Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"be111511-d320-425e-b9d8-63b03b10f264","seconds":300}
{"level":"info","ts":"2026-11-03T15:14:43.843Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"ae448bb8-63a2-4ea8-bd45-3615c8b5e068","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:14:43.843Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"ae448bb8-63a2-4ea8-bd45-3615c8b5e068","name":"long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:14:43.843Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"ae448bb8-63a2-4ea8-bd45-3615c8b5e068","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:14:43.843Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"ae448bb8-63a2-4ea8-bd45-3615c8b5e068"}
{"level":"info","ts":"2026-11-03T15:14:43.843Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"ae448bb8-63a2-4ea8-bd45-3615c8b5e068","RayCluster":"long-running-7qq77","State":""}
{"level":"info","ts":"2026-11-03T15:14:43.843Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"ae448bb8-63a2-4ea8-bd45-3615c8b5e068","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:40Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:40Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:14:46.834Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"16e7c424-0cbc-4cf6-b7da-a985da2e7fd8"}
{"level":"info","ts":"2026-11-03T15:14:46.834Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"16e7c424-0cbc-4cf6-b7da-a985da2e7fd8"}
{"level":"info","ts":"2026-11-03T15:14:46.834Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"16e7c424-0cbc-4cf6-b7da-a985da2e7fd8","1 head service found":"long-running-7qq77-head-svc"}
{"level":"info","ts":"2026-11-03T15:14:46.834Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"16e7c424-0cbc-4cf6-b7da-a985da2e7fd8","Found 1 head Pod":"long-running-7qq77-head-7xmdt","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:14:46.834Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"16e7c424-0cbc-4cf6-b7da-a985da2e7fd8","head Pod":"long-running-7qq77-head-7xmdt","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-7qq77-head-7xmdt. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:14:46.834Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"16e7c424-0cbc-4cf6-b7da-a985da2e7fd8","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:14:46.834Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"16e7c424-0cbc-4cf6-b7da-a985da2e7fd8","worker Pod":"long-running-7qq77-small-group-worker-6dvrl","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-7qq77-small-group-worker-6dvrl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:14:46.834Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"16e7c424-0cbc-4cf6-b7da-a985da2e7fd8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:14:46.834Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"16e7c424-0cbc-4cf6-b7da-a985da2e7fd8","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:14:46.834Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"16e7c424-0cbc-4cf6-b7da-a985da2e7fd8","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:14:46.834Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"16e7c424-0cbc-4cf6-b7da-a985da2e7fd8","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:14:46.834Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"16e7c424-0cbc-4cf6-b7da-a985da2e7fd8","podName":"long-running-7qq77-small-group-worker-6dvrl","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:13:18Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:14:46.834Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"16e7c424-0cbc-4cf6-b7da-a985da2e7fd8","podName":"long-running-7qq77-small-group-worker-6dvrl","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:13:18Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:14:46.835Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"16e7c424-0cbc-4cf6-b7da-a985da2e7fd8","name":"long-running-7qq77","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:40Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:14:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:14:46.844Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"f0597a06-53d7-4cc6-82b3-9ff9b247e9d4","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:14:46.844Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"f0597a06-53d7-4cc6-82b3-9ff9b247e9d4","name":"long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:14:46.844Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"f0597a06-53d7-4cc6-82b3-9ff9b247e9d4","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:14:46.844Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"f0597a06-53d7-4cc6-82b3-9ff9b247e9d4"}
{"level":"info","ts":"2026-11-03T15:14:46.844Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"f0597a06-53d7-4cc6-82b3-9ff9b247e9d4","RayCluster":"long-running-7qq77","State":""}
{"level":"info","ts":"2026-11-03T15:14:46.844Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"f0597a06-53d7-4cc6-82b3-9ff9b247e9d4","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:40Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:40Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:14:46.896Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"367235a6-880a-41cc-9730-4cff20b85c6c","cluster":"long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:14:46.896Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"b4dcef85-67cf-4fd8-b48d-8e3d0304b174","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:14:46.896Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"b4dcef85-67cf-4fd8-b48d-8e3d0304b174","name":"long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:14:46.896Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"b4dcef85-67cf-4fd8-b48d-8e3d0304b174","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:14:46.896Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"b4dcef85-67cf-4fd8-b48d-8e3d0304b174"}
{"level":"info","ts":"2026-11-03T15:14:46.896Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"b4dcef85-67cf-4fd8-b48d-8e3d0304b174","RayCluster":"long-running-7qq77","State":""}
{"level":"info","ts":"2026-11-03T15:14:46.896Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"b4dcef85-67cf-4fd8-b48d-8e3d0304b174","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:40Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:14:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:14:46.896Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"b4dcef85-67cf-4fd8-b48d-8e3d0304b174","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:14:46.958Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"3e91da68-e560-48c8-a188-3c07e72dcf9c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:14:46.958Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"3e91da68-e560-48c8-a188-3c07e72dcf9c","name":"long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:14:46.958Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"3e91da68-e560-48c8-a188-3c07e72dcf9c","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:14:46.958Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"3e91da68-e560-48c8-a188-3c07e72dcf9c"}
{"level":"info","ts":"2026-11-03T15:14:46.958Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"3e91da68-e560-48c8-a188-3c07e72dcf9c","RayCluster":"long-running-7qq77","State":""}
{"level":"info","ts":"2026-11-03T15:14:46.958Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"3e91da68-e560-48c8-a188-3c07e72dcf9c","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:14:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:14:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:14:48.896Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"eef8beb8-1c75-4bdf-b924-2ab3de9bf948"}
{"level":"info","ts":"2026-11-03T15:14:48.896Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"eef8beb8-1c75-4bdf-b924-2ab3de9bf948"}
{"level":"info","ts":"2026-11-03T15:14:48.896Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"eef8beb8-1c75-4bdf-b924-2ab3de9bf948","1 head service found":"long-running-7qq77-head-svc"}
{"level":"info","ts":"2026-11-03T15:14:48.896Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"eef8beb8-1c75-4bdf-b924-2ab3de9bf948","Found 1 head Pod":"long-running-7qq77-head-7xmdt","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:14:48.896Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"eef8beb8-1c75-4bdf-b924-2ab3de9bf948","head Pod":"long-running-7qq77-head-7xmdt","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-7qq77-head-7xmdt. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:14:48.896Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"eef8beb8-1c75-4bdf-b924-2ab3de9bf948","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:14:48.896Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"eef8beb8-1c75-4bdf-b924-2ab3de9bf948","worker Pod":"long-running-7qq77-small-group-worker-6dvrl","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-7qq77-small-group-worker-6dvrl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:14:48.896Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"eef8beb8-1c75-4bdf-b924-2ab3de9bf948","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:14:48.896Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"eef8beb8-1c75-4bdf-b924-2ab3de9bf948","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:14:48.896Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"eef8beb8-1c75-4bdf-b924-2ab3de9bf948","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:14:48.896Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"eef8beb8-1c75-4bdf-b924-2ab3de9bf948","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:14:48.897Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"eef8beb8-1c75-4bdf-b924-2ab3de9bf948","podName":"long-running-7qq77-small-group-worker-6dvrl","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:13:18Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:14:48.897Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"eef8beb8-1c75-4bdf-b924-2ab3de9bf948","podName":"long-running-7qq77-small-group-worker-6dvrl","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:13:18Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:14:48.897Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"eef8beb8-1c75-4bdf-b924-2ab3de9bf948","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:14:48.897Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"eef8beb8-1c75-4bdf-b924-2ab3de9bf948","seconds":300}
{"level":"info","ts":"2026-11-03T15:14:49.959Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"391d31d6-f5d6-4482-9a5b-349220d7c10f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:14:49.959Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"391d31d6-f5d6-4482-9a5b-349220d7c10f","name":"long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:14:49.959Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"391d31d6-f5d6-4482-9a5b-349220d7c10f","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:14:49.959Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"391d31d6-f5d6-4482-9a5b-349220d7c10f"}
{"level":"info","ts":"2026-11-03T15:14:49.959Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"391d31d6-f5d6-4482-9a5b-349220d7c10f","RayCluster":"long-running-7qq77","State":""}
{"level":"info","ts":"2026-11-03T15:14:49.959Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"391d31d6-f5d6-4482-9a5b-349220d7c10f","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:14:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:14:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:14:51.679Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"ad7b0408-7998-4446-9e8b-3b0f2b5274e5"}
{"level":"info","ts":"2026-11-03T15:14:51.679Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"ad7b0408-7998-4446-9e8b-3b0f2b5274e5"}
{"level":"info","ts":"2026-11-03T15:14:51.679Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"ad7b0408-7998-4446-9e8b-3b0f2b5274e5","1 head service found":"long-running-7qq77-head-svc"}
{"level":"info","ts":"2026-11-03T15:14:51.680Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"ad7b0408-7998-4446-9e8b-3b0f2b5274e5","Found 1 head Pod":"long-running-7qq77-head-7xmdt","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:14:51.680Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"ad7b0408-7998-4446-9e8b-3b0f2b5274e5","head Pod":"long-running-7qq77-head-7xmdt","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-7qq77-head-7xmdt. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:14:51.680Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"ad7b0408-7998-4446-9e8b-3b0f2b5274e5","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:14:51.680Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"ad7b0408-7998-4446-9e8b-3b0f2b5274e5","worker Pod":"long-running-7qq77-small-group-worker-6dvrl","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-7qq77-small-group-worker-6dvrl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:14:51.680Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"ad7b0408-7998-4446-9e8b-3b0f2b5274e5","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:14:51.680Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"ad7b0408-7998-4446-9e8b-3b0f2b5274e5","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:14:51.680Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"ad7b0408-7998-4446-9e8b-3b0f2b5274e5","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:14:51.680Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"ad7b0408-7998-4446-9e8b-3b0f2b5274e5","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:14:51.680Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"ad7b0408-7998-4446-9e8b-3b0f2b5274e5","name":"long-running-7qq77","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:14:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:51Z","stateTransitionTimes":{"ready":"2026-11-03T15:14:51Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:14:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:14:51Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:14:51.741Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"a4f9b3d6-17cb-4506-bc91-a9195e8758c0","cluster":"long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:14:51.741Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"5857e772-731a-49d2-91f5-e3f67b3b8c57","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:14:51.741Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"5857e772-731a-49d2-91f5-e3f67b3b8c57","name":"long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:14:51.741Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"5857e772-731a-49d2-91f5-e3f67b3b8c57","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:14:51.741Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"5857e772-731a-49d2-91f5-e3f67b3b8c57"}
{"level":"info","ts":"2026-11-03T15:14:51.804Z","logger":"controllers.RayJob","msg":"Created submitter Kubernetes Job for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"5857e772-731a-49d2-91f5-e3f67b3b8c57","Kubernetes Job":"long-running"}
{"level":"info","ts":"2026-11-03T15:14:51.804Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"5857e772-731a-49d2-91f5-e3f67b3b8c57","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:14:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","dashboardURL":"long-running-7qq77-head-svc.test-ns-5lqjm.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:14:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:14:51.804Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"5857e772-731a-49d2-91f5-e3f67b3b8c57","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-11-03T15:14:51.866Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"61c1eec0-1b87-4e79-8e08-e35966993150","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:14:51.866Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"61c1eec0-1b87-4e79-8e08-e35966993150","name":"long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:14:51.866Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"61c1eec0-1b87-4e79-8e08-e35966993150","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:14:51.866Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"61c1eec0-1b87-4e79-8e08-e35966993150"}
{"level":"info","ts":"2026-11-03T15:14:51.866Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job for RayJob already exists","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"61c1eec0-1b87-4e79-8e08-e35966993150","Kubernetes Job":"long-running"}
{"level":"info","ts":"2026-11-03T15:14:51.866Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"61c1eec0-1b87-4e79-8e08-e35966993150","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:14:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-xxbww","rayClusterName":"long-running-7qq77","dashboardURL":"long-running-7qq77-head-svc.test-ns-5lqjm.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-11-03T15:13:14Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.36","serviceIP":"10.132.0.36","podName":"long-running-7qq77-head-7xmdt","serviceName":"long-running-7qq77-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:14:46Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:13:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:13:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:14:51.866Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"61c1eec0-1b87-4e79-8e08-e35966993150","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-11-03T15:14:51.921Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"61c1eec0-1b87-4e79-8e08-e35966993150","error":"Operation cannot be fulfilled on rayjobs.ray.io \"long-running\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-11-03T15:14:51.921Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"61c1eec0-1b87-4e79-8e08-e35966993150"}
{"level":"error","ts":"2026-11-03T15:14:51.921Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"61c1eec0-1b87-4e79-8e08-e35966993150","error":"Operation cannot be fulfilled on rayjobs.ray.io \"long-running\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:14:51.921Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"7e06d5af-3882-45e2-a689-fe3748c3fd0f","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:14:51.921Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"7e06d5af-3882-45e2-a689-fe3748c3fd0f","name":"long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:14:51.921Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"7e06d5af-3882-45e2-a689-fe3748c3fd0f","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:14:51.921Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"7e06d5af-3882-45e2-a689-fe3748c3fd0f"}
{"level":"info","ts":"2026-11-03T15:14:51.921Z","logger":"controllers.RayJob","msg":"Update Kubernetes Service","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"7e06d5af-3882-45e2-a689-fe3748c3fd0f","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:14:51.985Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"05d80494-5abf-446b-83f6-4376839b158d","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:14:51.985Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"05d80494-5abf-446b-83f6-4376839b158d","name":"long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:14:51.985Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"05d80494-5abf-446b-83f6-4376839b158d","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:14:51.985Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"05d80494-5abf-446b-83f6-4376839b158d"}
{"level":"info","ts":"2026-11-03T15:14:51.985Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"05d80494-5abf-446b-83f6-4376839b158d","rayCluster":"long-running-7qq77","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:14:52.420Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"3ab1264f-23cd-47ca-b804-470a8fa41d6d","DeletionTimestamp":"2026-11-03 15:14:52 +0000 UTC"}
{"level":"error","ts":"2026-11-03T15:14:52.426Z","logger":"controllers.RayJob","msg":"Failed to stop job for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"3ab1264f-23cd-47ca-b804-470a8fa41d6d","error":"invalid character 'J' looking for beginning of value","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:128\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:14:52.427Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"3ab1264f-23cd-47ca-b804-470a8fa41d6d","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-11-03T15:14:52.489Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"0d7bc7de-cacc-4431-8a5f-d4c32cd9e4b0","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:14:52.489Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"0d7bc7de-cacc-4431-8a5f-d4c32cd9e4b0","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-11-03T15:14:52.510Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"long-running-7qq77","reconcileID":"7310db5b-3c2a-4bf8-94f7-7a725fb1fcb7","namespacedName":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:14:52.510Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"long-running-7qq77","reconcileID":"7310db5b-3c2a-4bf8-94f7-7a725fb1fcb7","cluster":"long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:14:52.510Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"long-running-7qq77","reconcileID":"7310db5b-3c2a-4bf8-94f7-7a725fb1fcb7","cluster":"long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:14:52.510Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"reconcileID":"c2f8d3d8-7bee-4c99-b368-91cc1c4403df"}
{"level":"info","ts":"2026-11-03T15:14:52.510Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"long-running-7qq77","reconcileID":"7310db5b-3c2a-4bf8-94f7-7a725fb1fcb7","namespace":"test-ns-5lqjm","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-11-03T15:14:52.547Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"0d7bc7de-cacc-4431-8a5f-d4c32cd9e4b0"}
{"level":"info","ts":"2026-11-03T15:14:52.547Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"0d7bc7de-cacc-4431-8a5f-d4c32cd9e4b0","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-11-03T15:14:52.547Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"0d7bc7de-cacc-4431-8a5f-d4c32cd9e4b0","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-krfqf","rayClusterName":"counter-gkt5t","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:52Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:14:52.547Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"0d7bc7de-cacc-4431-8a5f-d4c32cd9e4b0","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:14:52.562Z","logger":"authentication-controller","msg":"Deleted shared ReferenceGrant (last cluster with auth)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"long-running-7qq77","reconcileID":"7310db5b-3c2a-4bf8-94f7-7a725fb1fcb7","name":"kuberay-gateway-access","namespace":"test-ns-5lqjm"}
{"level":"info","ts":"2026-11-03T15:14:52.606Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"266e86a5-1aa7-40d4-801a-7d6bc8c7351c"}
{"level":"info","ts":"2026-11-03T15:14:52.606Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"838856f8-cd9b-40f1-8af7-4c3e7187b32d","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:14:52.606Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"838856f8-cd9b-40f1-8af7-4c3e7187b32d"}
{"level":"info","ts":"2026-11-03T15:14:52.606Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"838856f8-cd9b-40f1-8af7-4c3e7187b32d","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-11-03T15:14:52.606Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"838856f8-cd9b-40f1-8af7-4c3e7187b32d","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-5dnfh","rayClusterName":"counter-9ctkc","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:52Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:14:52.606Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"838856f8-cd9b-40f1-8af7-4c3e7187b32d","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:14:52.614Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"long-running-7qq77","reconcileID":"7310db5b-3c2a-4bf8-94f7-7a725fb1fcb7","name":"test-ns-5lqjm-long-running-7qq77","namespace":"opendatahub"}
{"level":"info","ts":"2026-11-03T15:14:52.659Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"838856f8-cd9b-40f1-8af7-4c3e7187b32d","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-11-03T15:14:52.659Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"838856f8-cd9b-40f1-8af7-4c3e7187b32d"}
{"level":"error","ts":"2026-11-03T15:14:52.659Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"838856f8-cd9b-40f1-8af7-4c3e7187b32d","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:14:52.659Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"9c51248c-22e7-48ad-b8fc-c82c11925cb5","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:14:52.659Z","logger":"controllers.RayJob","msg":"Try to transition the status to `Suspending`","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"9c51248c-22e7-48ad-b8fc-c82c11925cb5","oldStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:14:52.659Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"9c51248c-22e7-48ad-b8fc-c82c11925cb5","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-krfqf","rayClusterName":"counter-gkt5t","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:52Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-krfqf","rayClusterName":"counter-gkt5t","jobDeploymentStatus":"Suspending","startTime":"2026-11-03T15:14:52Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:14:52.660Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"9c51248c-22e7-48ad-b8fc-c82c11925cb5","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Suspending"}
{"level":"info","ts":"2026-11-03T15:14:52.666Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"long-running-7qq77","reconcileID":"7310db5b-3c2a-4bf8-94f7-7a725fb1fcb7","configMap":"kube-rbac-proxy-config-long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:14:52.718Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"6d25dbb4-5227-472c-9642-870ff6524dbd","JobStatus":"","JobDeploymentStatus":"Suspending","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:14:52.718Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"6d25dbb4-5227-472c-9642-870ff6524dbd","RayCluster":{"name":"counter-gkt5t","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:14:52.718Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"6d25dbb4-5227-472c-9642-870ff6524dbd","isClusterDeleted":true}
{"level":"info","ts":"2026-11-03T15:14:52.718Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job has been already deleted","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"6d25dbb4-5227-472c-9642-870ff6524dbd","Kubernetes Job":""}
{"level":"info","ts":"2026-11-03T15:14:52.718Z","logger":"controllers.RayJob","msg":"deleteSubmitterJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"6d25dbb4-5227-472c-9642-870ff6524dbd","isJobDeleted":true}
{"level":"info","ts":"2026-11-03T15:14:52.718Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"6d25dbb4-5227-472c-9642-870ff6524dbd","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-krfqf","rayClusterName":"counter-gkt5t","jobDeploymentStatus":"Suspending","startTime":"2026-11-03T15:14:52Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobDeploymentStatus":"Suspended","startTime":"2026-11-03T15:14:52Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:14:52.718Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"6d25dbb4-5227-472c-9642-870ff6524dbd","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Suspending","new JobDeploymentStatus":"Suspended"}
{"level":"info","ts":"2026-11-03T15:14:52.719Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"long-running-7qq77","reconcileID":"7310db5b-3c2a-4bf8-94f7-7a725fb1fcb7","serviceAccount":"long-running-7qq77-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-11-03T15:14:52.778Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"dcd13a61-b58f-4663-8f10-b16ff35cb3b4","JobStatus":"","JobDeploymentStatus":"Suspended","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:14:52.910Z","logger":"raycluster-resource","msg":"default","name":"long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:14:52.910Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"long-running-7qq77","namespace":"test-ns-5lqjm"}
{"level":"info","ts":"2026-11-03T15:14:53.090Z","logger":"raycluster-resource","msg":"validate update","name":"long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:14:53.145Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"long-running-7qq77","reconcileID":"7310db5b-3c2a-4bf8-94f7-7a725fb1fcb7","cluster":"long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:14:53.145Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"long-running-7qq77","reconcileID":"c03ee6b6-0ddd-4939-a450-273118147e66","namespacedName":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:14:53.145Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"long-running-7qq77","reconcileID":"c03ee6b6-0ddd-4939-a450-273118147e66","cluster":"long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:14:53.145Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"long-running-7qq77","reconcileID":"c03ee6b6-0ddd-4939-a450-273118147e66","cluster":"long-running-7qq77"}
{"level":"info","ts":"2026-11-03T15:14:53.145Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"long-running-7qq77","reconcileID":"c03ee6b6-0ddd-4939-a450-273118147e66","namespace":"test-ns-5lqjm","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-11-03T15:14:53.145Z","logger":"authentication-controller","msg":"ReferenceGrant already deleted","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"long-running-7qq77","reconcileID":"c03ee6b6-0ddd-4939-a450-273118147e66","name":"kuberay-gateway-access"}
{"level":"info","ts":"2026-11-03T15:14:53.145Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"long-running","namespace":"test-ns-5lqjm"},"reconcileID":"893b6f3f-1ed7-4fb0-b693-7f22649741d9"}
{"level":"error","ts":"2026-11-03T15:14:53.198Z","logger":"authentication-controller","msg":"Failed to remove finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"long-running-7qq77","reconcileID":"c03ee6b6-0ddd-4939-a450-273118147e66","error":"rayclusters.ray.io \"long-running-7qq77\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleDeletion\n\t/workspace/controllers/ray/authentication_controller.go:195\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:14:53.198Z","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"long-running-7qq77","reconcileID":"c03ee6b6-0ddd-4939-a450-273118147e66"}
{"level":"error","ts":"2026-11-03T15:14:53.198Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"long-running-7qq77","reconcileID":"c03ee6b6-0ddd-4939-a450-273118147e66","error":"rayclusters.ray.io \"long-running-7qq77\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:14:53.198Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"long-running-7qq77","reconcileID":"c05ca4db-d17d-4478-8738-1a85b5af0a87","namespacedName":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:14:53.198Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"long-running-7qq77","reconcileID":"c05ca4db-d17d-4478-8738-1a85b5af0a87"}
{"level":"info","ts":"2026-11-03T15:14:53.198Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"long-running-7qq77","reconcileID":"c05ca4db-d17d-4478-8738-1a85b5af0a87","namespace":"test-ns-5lqjm","authClusters":0}
{"level":"info","ts":"2026-11-03T15:14:53.198Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-7qq77","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"long-running-7qq77","reconcileID":"c05ca4db-d17d-4478-8738-1a85b5af0a87","namespace":"test-ns-5lqjm"}
{"level":"info","ts":"2026-11-03T15:14:53.505Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"98c9bf85-3491-4515-92f9-dc18ba97a92d","JobStatus":"","JobDeploymentStatus":"Suspended","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:14:53.505Z","logger":"controllers.RayJob","msg":"The status is 'Suspended', but the suspend flag is false. Transition the status to 'New'.","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"98c9bf85-3491-4515-92f9-dc18ba97a92d"}
{"level":"info","ts":"2026-11-03T15:14:53.505Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"98c9bf85-3491-4515-92f9-dc18ba97a92d","oldRayJobStatus":{"rayJobInfo":{},"jobDeploymentStatus":"Suspended","startTime":"2026-11-03T15:14:52Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"startTime":"2026-11-03T15:14:52Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:14:53.505Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"98c9bf85-3491-4515-92f9-dc18ba97a92d","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Suspended","new JobDeploymentStatus":""}
{"level":"info","ts":"2026-11-03T15:14:53.567Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"c22c3f2b-6a73-4fbf-9fe2-33a0c01674cd","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:14:53.567Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"c22c3f2b-6a73-4fbf-9fe2-33a0c01674cd"}
{"level":"info","ts":"2026-11-03T15:14:53.567Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"c22c3f2b-6a73-4fbf-9fe2-33a0c01674cd","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-11-03T15:14:53.567Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"c22c3f2b-6a73-4fbf-9fe2-33a0c01674cd","oldRayJobStatus":{"rayJobInfo":{},"startTime":"2026-11-03T15:14:52Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:14:53.567Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"c22c3f2b-6a73-4fbf-9fe2-33a0c01674cd","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:14:53.626Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"c4f477bb-29ce-47df-b4fc-151088862e8e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:14:53.626Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"c4f477bb-29ce-47df-b4fc-151088862e8e","name":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:14:53.626Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"c4f477bb-29ce-47df-b4fc-151088862e8e","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:14:53.626Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"c4f477bb-29ce-47df-b4fc-151088862e8e","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:14:53.676Z","logger":"raycluster-resource","msg":"default","name":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:14:53.676Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"counter-8j287","namespace":"test-ns-5lqjm"}
{"level":"info","ts":"2026-11-03T15:14:53.734Z","logger":"raycluster-resource","msg":"validate create","name":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:14:53.788Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"c4f477bb-29ce-47df-b4fc-151088862e8e","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:14:53.788Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"78f13e84-159c-4b80-a992-c304b93143d0"}
{"level":"info","ts":"2026-11-03T15:14:53.788Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"78f13e84-159c-4b80-a992-c304b93143d0"}
{"level":"info","ts":"2026-11-03T15:14:53.788Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"97ee67e0-85d1-4de9-b326-45ba8146e33c","cluster":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:14:53.788Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"counter-8j287","reconcileID":"d9a735b1-c8a8-4bdc-84e0-2b3e0cb6b926","namespacedName":{"name":"counter-8j287","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:14:53.788Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"counter-8j287","reconcileID":"d9a735b1-c8a8-4bdc-84e0-2b3e0cb6b926","mode":"IntegratedOAuth","cluster":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:14:53.788Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"counter-8j287","reconcileID":"d9a735b1-c8a8-4bdc-84e0-2b3e0cb6b926","cluster":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:14:53.789Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"counter-8j287","reconcileID":"d9a735b1-c8a8-4bdc-84e0-2b3e0cb6b926","cluster":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:14:53.789Z","logger":"controllers.NetworkPolicy","msg":"Reconciling NetworkPolicies for RayCluster","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"02a64755-bf8e-41b2-b530-015a117ef63a","cluster":"counter-8j287","namespace":"test-ns-5lqjm"}
{"level":"info","ts":"2026-11-03T15:14:53.788Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"c4f477bb-29ce-47df-b4fc-151088862e8e"}
{"level":"info","ts":"2026-11-03T15:14:53.789Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"c4f477bb-29ce-47df-b4fc-151088862e8e","RayCluster":"counter-8j287","State":""}
{"level":"info","ts":"2026-11-03T15:14:53.789Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"c4f477bb-29ce-47df-b4fc-151088862e8e","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:14:53.789Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"cb6bf429-6a41-4803-8c4e-77e26b246a55","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:14:53.789Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"cb6bf429-6a41-4803-8c4e-77e26b246a55","name":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:14:53.789Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"cb6bf429-6a41-4803-8c4e-77e26b246a55","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:14:53.789Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"cb6bf429-6a41-4803-8c4e-77e26b246a55"}
{"level":"info","ts":"2026-11-03T15:14:53.789Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"cb6bf429-6a41-4803-8c4e-77e26b246a55","RayCluster":"counter-8j287","State":""}
{"level":"info","ts":"2026-11-03T15:14:53.789Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"cb6bf429-6a41-4803-8c4e-77e26b246a55","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:14:53.841Z","logger":"raycluster-resource","msg":"default","name":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:14:53.841Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"counter-8j287","namespace":"test-ns-5lqjm"}
{"level":"info","ts":"2026-11-03T15:14:53.843Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"78f13e84-159c-4b80-a992-c304b93143d0","name":"counter-8j287-head-svc"}
{"level":"info","ts":"2026-11-03T15:14:53.843Z","logger":"controllers.NetworkPolicy","msg":"Successfully created NetworkPolicy","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"02a64755-bf8e-41b2-b530-015a117ef63a","name":"counter-8j287-head"}
{"level":"info","ts":"2026-11-03T15:14:53.843Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"78f13e84-159c-4b80-a992-c304b93143d0"}
{"level":"info","ts":"2026-11-03T15:14:53.892Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"78f13e84-159c-4b80-a992-c304b93143d0","error":"certificate ray-head-cert-counter-8j287 not found: certificates.cert-manager.io \"ray-head-cert-counter-8j287\" not found"}
{"level":"error","ts":"2026-11-03T15:14:53.892Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"78f13e84-159c-4b80-a992-c304b93143d0","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-counter-8j287 not found: certificates.cert-manager.io \"ray-head-cert-counter-8j287\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:14:53.892Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"78f13e84-159c-4b80-a992-c304b93143d0","error":"mTLS secrets not ready: certificate ray-head-cert-counter-8j287 not found: certificates.cert-manager.io \"ray-head-cert-counter-8j287\" not found"}
{"level":"info","ts":"2026-11-03T15:14:53.892Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"78f13e84-159c-4b80-a992-c304b93143d0","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"counter-8j287","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:14:53.892Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"78f13e84-159c-4b80-a992-c304b93143d0","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"counter-8j287","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:14:53.892Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"78f13e84-159c-4b80-a992-c304b93143d0","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"counter-8j287","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:14:53.892Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"78f13e84-159c-4b80-a992-c304b93143d0","name":"counter-8j287","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:14:53.897Z","logger":"controllers.NetworkPolicy","msg":"Successfully created NetworkPolicy","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"02a64755-bf8e-41b2-b530-015a117ef63a","name":"counter-8j287-workers-small-group"}
{"level":"info","ts":"2026-11-03T15:14:53.897Z","logger":"controllers.NetworkPolicy","msg":"Successfully reconciled NetworkPolicies for RayCluster","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"02a64755-bf8e-41b2-b530-015a117ef63a","cluster":"counter-8j287","namespace":"test-ns-5lqjm"}
{"level":"info","ts":"2026-11-03T15:14:53.897Z","logger":"controllers.NetworkPolicy","msg":"Reconciling NetworkPolicies for RayCluster","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"1b5a52e2-161f-4dc8-bb18-a2df9fc2727c","cluster":"counter-8j287","namespace":"test-ns-5lqjm"}
{"level":"info","ts":"2026-11-03T15:14:53.898Z","logger":"raycluster-resource","msg":"validate update","name":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:14:53.951Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"78f13e84-159c-4b80-a992-c304b93143d0"}
{"level":"error","ts":"2026-11-03T15:14:53.951Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"78f13e84-159c-4b80-a992-c304b93143d0","error":"mTLS secrets not ready: certificate ray-head-cert-counter-8j287 not found: certificates.cert-manager.io \"ray-head-cert-counter-8j287\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:14:53.951Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"ef0d8631-d35e-4422-b325-0aad84ce809a"}
{"level":"info","ts":"2026-11-03T15:14:53.951Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"ef0d8631-d35e-4422-b325-0aad84ce809a"}
{"level":"info","ts":"2026-11-03T15:14:53.951Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"ef0d8631-d35e-4422-b325-0aad84ce809a","1 head service found":"counter-8j287-head-svc"}
{"level":"info","ts":"2026-11-03T15:14:53.951Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"ef0d8631-d35e-4422-b325-0aad84ce809a"}
{"level":"info","ts":"2026-11-03T15:14:53.951Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"40175029-2ae7-4b83-958a-1d2f6793915c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:14:53.951Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"40175029-2ae7-4b83-958a-1d2f6793915c","name":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:14:53.951Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"40175029-2ae7-4b83-958a-1d2f6793915c","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:14:53.952Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"40175029-2ae7-4b83-958a-1d2f6793915c"}
{"level":"info","ts":"2026-11-03T15:14:53.952Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"40175029-2ae7-4b83-958a-1d2f6793915c","RayCluster":"counter-8j287","State":""}
{"level":"info","ts":"2026-11-03T15:14:53.952Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"40175029-2ae7-4b83-958a-1d2f6793915c","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:14:53.952Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"40175029-2ae7-4b83-958a-1d2f6793915c","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:14:53.956Z","logger":"raycluster-resource","msg":"default","name":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:14:53.956Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"counter-8j287","namespace":"test-ns-5lqjm"}
{"level":"info","ts":"2026-11-03T15:14:54.001Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"ef0d8631-d35e-4422-b325-0aad84ce809a","error":"certificate ray-head-cert-counter-8j287 not found: certificates.cert-manager.io \"ray-head-cert-counter-8j287\" not found"}
{"level":"error","ts":"2026-11-03T15:14:54.001Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"ef0d8631-d35e-4422-b325-0aad84ce809a","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-counter-8j287 not found: certificates.cert-manager.io \"ray-head-cert-counter-8j287\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:14:54.001Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"ef0d8631-d35e-4422-b325-0aad84ce809a","error":"mTLS secrets not ready: certificate ray-head-cert-counter-8j287 not found: certificates.cert-manager.io \"ray-head-cert-counter-8j287\" not found"}
{"level":"info","ts":"2026-11-03T15:14:54.001Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"ef0d8631-d35e-4422-b325-0aad84ce809a","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"counter-8j287","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:14:54.001Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"ef0d8631-d35e-4422-b325-0aad84ce809a","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"counter-8j287","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:14:54.001Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"ef0d8631-d35e-4422-b325-0aad84ce809a","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"counter-8j287","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:14:54.001Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"ef0d8631-d35e-4422-b325-0aad84ce809a","name":"counter-8j287","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:54Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:54Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:54Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:54Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"error","ts":"2026-11-03T15:14:54.004Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"counter-8j287","reconcileID":"d9a735b1-c8a8-4bdc-84e0-2b3e0cb6b926","error":"failed to add finalizer: Operation cannot be fulfilled on rayclusters.ray.io \"counter-8j287\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:167\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:14:54.004Z","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"counter-8j287","reconcileID":"d9a735b1-c8a8-4bdc-84e0-2b3e0cb6b926"}
{"level":"error","ts":"2026-11-03T15:14:54.004Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"counter-8j287","reconcileID":"d9a735b1-c8a8-4bdc-84e0-2b3e0cb6b926","error":"failed to add finalizer: Operation cannot be fulfilled on rayclusters.ray.io \"counter-8j287\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:14:54.004Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"counter-8j287","reconcileID":"3562df6c-a0a7-4a3a-b076-755d699a7f53","namespacedName":{"name":"counter-8j287","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:14:54.004Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"counter-8j287","reconcileID":"3562df6c-a0a7-4a3a-b076-755d699a7f53","mode":"IntegratedOAuth","cluster":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:14:54.004Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"counter-8j287","reconcileID":"3562df6c-a0a7-4a3a-b076-755d699a7f53","cluster":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:14:54.004Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"counter-8j287","reconcileID":"3562df6c-a0a7-4a3a-b076-755d699a7f53","cluster":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:14:54.009Z","logger":"controllers.NetworkPolicy","msg":"Successfully reconciled NetworkPolicies for RayCluster","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"1b5a52e2-161f-4dc8-bb18-a2df9fc2727c","cluster":"counter-8j287","namespace":"test-ns-5lqjm"}
{"level":"info","ts":"2026-11-03T15:14:54.012Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"4e20f38f-ac04-49c0-b459-670ac942cb25","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:14:54.012Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"4e20f38f-ac04-49c0-b459-670ac942cb25","name":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:14:54.012Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"4e20f38f-ac04-49c0-b459-670ac942cb25","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:14:54.012Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"4e20f38f-ac04-49c0-b459-670ac942cb25"}
{"level":"info","ts":"2026-11-03T15:14:54.012Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"4e20f38f-ac04-49c0-b459-670ac942cb25","RayCluster":"counter-8j287","State":""}
{"level":"info","ts":"2026-11-03T15:14:54.012Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"4e20f38f-ac04-49c0-b459-670ac942cb25","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:14:54.055Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"ef0d8631-d35e-4422-b325-0aad84ce809a","name":"counter-8j287","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-8j287\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-5lqjm","name":"counter-8j287"}}
{"level":"info","ts":"2026-11-03T15:14:54.055Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"ef0d8631-d35e-4422-b325-0aad84ce809a"}
{"level":"error","ts":"2026-11-03T15:14:54.055Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"ef0d8631-d35e-4422-b325-0aad84ce809a","error":"mTLS secrets not ready: certificate ray-head-cert-counter-8j287 not found: certificates.cert-manager.io \"ray-head-cert-counter-8j287\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:14:54.057Z","logger":"raycluster-resource","msg":"default","name":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:14:54.057Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"counter-8j287","namespace":"test-ns-5lqjm"}
{"level":"info","ts":"2026-11-03T15:14:54.066Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"cd794651-d51a-4a6e-8f8e-47917e42398c"}
{"level":"info","ts":"2026-11-03T15:14:54.066Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"cd794651-d51a-4a6e-8f8e-47917e42398c"}
{"level":"info","ts":"2026-11-03T15:14:54.066Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"cd794651-d51a-4a6e-8f8e-47917e42398c","1 head service found":"counter-8j287-head-svc"}
{"level":"info","ts":"2026-11-03T15:14:54.066Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"cd794651-d51a-4a6e-8f8e-47917e42398c"}
{"level":"info","ts":"2026-11-03T15:14:54.115Z","logger":"raycluster-resource","msg":"validate update","name":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:14:54.115Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"cd794651-d51a-4a6e-8f8e-47917e42398c","error":"certificate ray-head-cert-counter-8j287 not found: certificates.cert-manager.io \"ray-head-cert-counter-8j287\" not found"}
{"level":"error","ts":"2026-11-03T15:14:54.115Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"cd794651-d51a-4a6e-8f8e-47917e42398c","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-counter-8j287 not found: certificates.cert-manager.io \"ray-head-cert-counter-8j287\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:14:54.115Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"cd794651-d51a-4a6e-8f8e-47917e42398c","error":"mTLS secrets not ready: certificate ray-head-cert-counter-8j287 not found: certificates.cert-manager.io \"ray-head-cert-counter-8j287\" not found"}
{"level":"info","ts":"2026-11-03T15:14:54.115Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"cd794651-d51a-4a6e-8f8e-47917e42398c","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"counter-8j287","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:14:54.115Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"cd794651-d51a-4a6e-8f8e-47917e42398c","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"counter-8j287","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:14:54.115Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"cd794651-d51a-4a6e-8f8e-47917e42398c","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"counter-8j287","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:14:54.115Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"cd794651-d51a-4a6e-8f8e-47917e42398c"}
{"level":"error","ts":"2026-11-03T15:14:54.115Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"cd794651-d51a-4a6e-8f8e-47917e42398c","error":"mTLS secrets not ready: certificate ray-head-cert-counter-8j287 not found: certificates.cert-manager.io \"ray-head-cert-counter-8j287\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:14:54.136Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"a4eb5ea3-c178-4d55-ad24-ef200444cb84"}
{"level":"info","ts":"2026-11-03T15:14:54.136Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"a4eb5ea3-c178-4d55-ad24-ef200444cb84"}
{"level":"info","ts":"2026-11-03T15:14:54.136Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"a4eb5ea3-c178-4d55-ad24-ef200444cb84","1 head service found":"counter-8j287-head-svc"}
{"level":"info","ts":"2026-11-03T15:14:54.136Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"a4eb5ea3-c178-4d55-ad24-ef200444cb84"}
{"level":"info","ts":"2026-11-03T15:14:54.170Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"852a8ef8-1e12-41c1-959e-dd7e40895ac3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:14:54.170Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"852a8ef8-1e12-41c1-959e-dd7e40895ac3","name":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:14:54.170Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"852a8ef8-1e12-41c1-959e-dd7e40895ac3","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:14:54.170Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"852a8ef8-1e12-41c1-959e-dd7e40895ac3"}
{"level":"info","ts":"2026-11-03T15:14:54.170Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"852a8ef8-1e12-41c1-959e-dd7e40895ac3","RayCluster":"counter-8j287","State":""}
{"level":"info","ts":"2026-11-03T15:14:54.170Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"852a8ef8-1e12-41c1-959e-dd7e40895ac3","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:14:54.185Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"a4eb5ea3-c178-4d55-ad24-ef200444cb84","error":"certificate ray-head-cert-counter-8j287 not found: certificates.cert-manager.io \"ray-head-cert-counter-8j287\" not found"}
{"level":"error","ts":"2026-11-03T15:14:54.185Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"a4eb5ea3-c178-4d55-ad24-ef200444cb84","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-counter-8j287 not found: certificates.cert-manager.io \"ray-head-cert-counter-8j287\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:14:54.185Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"a4eb5ea3-c178-4d55-ad24-ef200444cb84","error":"mTLS secrets not ready: certificate ray-head-cert-counter-8j287 not found: certificates.cert-manager.io \"ray-head-cert-counter-8j287\" not found"}
{"level":"info","ts":"2026-11-03T15:14:54.185Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"a4eb5ea3-c178-4d55-ad24-ef200444cb84","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"counter-8j287","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:14:54.185Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"a4eb5ea3-c178-4d55-ad24-ef200444cb84","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"counter-8j287","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:14:54.185Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"a4eb5ea3-c178-4d55-ad24-ef200444cb84","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"counter-8j287","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:14:54.185Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"a4eb5ea3-c178-4d55-ad24-ef200444cb84"}
{"level":"error","ts":"2026-11-03T15:14:54.185Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"a4eb5ea3-c178-4d55-ad24-ef200444cb84","error":"mTLS secrets not ready: certificate ray-head-cert-counter-8j287 not found: certificates.cert-manager.io \"ray-head-cert-counter-8j287\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:14:54.223Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"counter-8j287","reconcileID":"3562df6c-a0a7-4a3a-b076-755d699a7f53","name":"counter-8j287-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-11-03T15:14:54.226Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"096279eb-5a9c-4b34-a916-30626eddd5ce"}
{"level":"info","ts":"2026-11-03T15:14:54.226Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"096279eb-5a9c-4b34-a916-30626eddd5ce"}
{"level":"info","ts":"2026-11-03T15:14:54.226Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"096279eb-5a9c-4b34-a916-30626eddd5ce","1 head service found":"counter-8j287-head-svc"}
{"level":"info","ts":"2026-11-03T15:14:54.226Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"096279eb-5a9c-4b34-a916-30626eddd5ce"}
{"level":"info","ts":"2026-11-03T15:14:54.275Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"counter-8j287","reconcileID":"3562df6c-a0a7-4a3a-b076-755d699a7f53","name":"kuberay-gateway-access","namespace":"test-ns-5lqjm","operation":"created"}
{"level":"info","ts":"2026-11-03T15:14:54.275Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"096279eb-5a9c-4b34-a916-30626eddd5ce","error":"certificate ray-head-cert-counter-8j287 not found: certificates.cert-manager.io \"ray-head-cert-counter-8j287\" not found"}
{"level":"error","ts":"2026-11-03T15:14:54.275Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"096279eb-5a9c-4b34-a916-30626eddd5ce","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-counter-8j287 not found: certificates.cert-manager.io \"ray-head-cert-counter-8j287\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:14:54.275Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"096279eb-5a9c-4b34-a916-30626eddd5ce","error":"mTLS secrets not ready: certificate ray-head-cert-counter-8j287 not found: certificates.cert-manager.io \"ray-head-cert-counter-8j287\" not found"}
{"level":"info","ts":"2026-11-03T15:14:54.276Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"096279eb-5a9c-4b34-a916-30626eddd5ce","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"counter-8j287","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:14:54.276Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"096279eb-5a9c-4b34-a916-30626eddd5ce","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"counter-8j287","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:14:54.276Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"096279eb-5a9c-4b34-a916-30626eddd5ce","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"counter-8j287","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:14:54.276Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"096279eb-5a9c-4b34-a916-30626eddd5ce"}
{"level":"error","ts":"2026-11-03T15:14:54.276Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"096279eb-5a9c-4b34-a916-30626eddd5ce","error":"mTLS secrets not ready: certificate ray-head-cert-counter-8j287 not found: certificates.cert-manager.io \"ray-head-cert-counter-8j287\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:14:54.330Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"counter-8j287","reconcileID":"3562df6c-a0a7-4a3a-b076-755d699a7f53","name":"test-ns-5lqjm-counter-8j287","operation":"created"}
{"level":"info","ts":"2026-11-03T15:14:54.357Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"ccb07935-af11-43a1-98dc-c4fb57106587"}
{"level":"info","ts":"2026-11-03T15:14:54.357Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"ccb07935-af11-43a1-98dc-c4fb57106587"}
{"level":"info","ts":"2026-11-03T15:14:54.357Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"ccb07935-af11-43a1-98dc-c4fb57106587","1 head service found":"counter-8j287-head-svc"}
{"level":"info","ts":"2026-11-03T15:14:54.357Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"ccb07935-af11-43a1-98dc-c4fb57106587"}
{"level":"info","ts":"2026-11-03T15:14:54.386Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"counter-8j287","reconcileID":"3562df6c-a0a7-4a3a-b076-755d699a7f53","name":"kube-rbac-proxy-config-counter-8j287","operation":"created"}
{"level":"info","ts":"2026-11-03T15:14:54.412Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"ccb07935-af11-43a1-98dc-c4fb57106587","error":"certificate ray-head-cert-counter-8j287 not found: certificates.cert-manager.io \"ray-head-cert-counter-8j287\" not found"}
{"level":"error","ts":"2026-11-03T15:14:54.412Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"ccb07935-af11-43a1-98dc-c4fb57106587","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-counter-8j287 not found: certificates.cert-manager.io \"ray-head-cert-counter-8j287\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:14:54.412Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"ccb07935-af11-43a1-98dc-c4fb57106587","error":"mTLS secrets not ready: certificate ray-head-cert-counter-8j287 not found: certificates.cert-manager.io \"ray-head-cert-counter-8j287\" not found"}
{"level":"info","ts":"2026-11-03T15:14:54.412Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"ccb07935-af11-43a1-98dc-c4fb57106587","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"counter-8j287","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:14:54.412Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"ccb07935-af11-43a1-98dc-c4fb57106587","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"counter-8j287","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:14:54.412Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"ccb07935-af11-43a1-98dc-c4fb57106587","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"counter-8j287","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:14:54.412Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"ccb07935-af11-43a1-98dc-c4fb57106587"}
{"level":"error","ts":"2026-11-03T15:14:54.412Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"ccb07935-af11-43a1-98dc-c4fb57106587","error":"mTLS secrets not ready: certificate ray-head-cert-counter-8j287 not found: certificates.cert-manager.io \"ray-head-cert-counter-8j287\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-03T15:14:54.444Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"counter-8j287","reconcileID":"3562df6c-a0a7-4a3a-b076-755d699a7f53","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"counter-8j287\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:167\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:14:54.444Z","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"counter-8j287","reconcileID":"3562df6c-a0a7-4a3a-b076-755d699a7f53"}
{"level":"error","ts":"2026-11-03T15:14:54.444Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"counter-8j287","reconcileID":"3562df6c-a0a7-4a3a-b076-755d699a7f53","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"counter-8j287\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:14:54.444Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"counter-8j287","reconcileID":"39dda2e0-d121-4598-83ae-b4d94da2c296","namespacedName":{"name":"counter-8j287","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:14:54.444Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"counter-8j287","reconcileID":"39dda2e0-d121-4598-83ae-b4d94da2c296","mode":"IntegratedOAuth","cluster":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:14:54.444Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"counter-8j287","reconcileID":"39dda2e0-d121-4598-83ae-b4d94da2c296","cluster":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:14:54.504Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"counter-8j287","reconcileID":"39dda2e0-d121-4598-83ae-b4d94da2c296","cluster":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:14:54.505Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"7c1ec332-8c3a-4e09-b0f2-eb91e739b413","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:14:54.505Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"7c1ec332-8c3a-4e09-b0f2-eb91e739b413","name":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:14:54.505Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"7c1ec332-8c3a-4e09-b0f2-eb91e739b413","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:14:54.505Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"7c1ec332-8c3a-4e09-b0f2-eb91e739b413"}
{"level":"info","ts":"2026-11-03T15:14:54.505Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"7c1ec332-8c3a-4e09-b0f2-eb91e739b413","RayCluster":"counter-8j287","State":""}
{"level":"info","ts":"2026-11-03T15:14:54.505Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"7c1ec332-8c3a-4e09-b0f2-eb91e739b413","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:14:54.505Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"7c1ec332-8c3a-4e09-b0f2-eb91e739b413","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:14:54.566Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"03dfddf0-9b6a-4ee9-a5d4-cab7e0b3f36c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:14:54.566Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"03dfddf0-9b6a-4ee9-a5d4-cab7e0b3f36c","name":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:14:54.566Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"03dfddf0-9b6a-4ee9-a5d4-cab7e0b3f36c","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:14:54.566Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"03dfddf0-9b6a-4ee9-a5d4-cab7e0b3f36c"}
{"level":"info","ts":"2026-11-03T15:14:54.566Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"03dfddf0-9b6a-4ee9-a5d4-cab7e0b3f36c","RayCluster":"counter-8j287","State":""}
{"level":"info","ts":"2026-11-03T15:14:54.566Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"03dfddf0-9b6a-4ee9-a5d4-cab7e0b3f36c","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:14:54.573Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"a620b995-97b0-4ad1-b08d-7fdc0fcb6bab"}
{"level":"info","ts":"2026-11-03T15:14:54.573Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"a620b995-97b0-4ad1-b08d-7fdc0fcb6bab"}
{"level":"info","ts":"2026-11-03T15:14:54.573Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"a620b995-97b0-4ad1-b08d-7fdc0fcb6bab","1 head service found":"counter-8j287-head-svc"}
{"level":"info","ts":"2026-11-03T15:14:54.573Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"a620b995-97b0-4ad1-b08d-7fdc0fcb6bab"}
{"level":"info","ts":"2026-11-03T15:14:54.621Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"97ee67e0-85d1-4de9-b326-45ba8146e33c","certificate":"ray-head-cert-counter-8j287"}
{"level":"info","ts":"2026-11-03T15:14:54.621Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"97ee67e0-85d1-4de9-b326-45ba8146e33c"}
{"level":"info","ts":"2026-11-03T15:14:54.621Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"2841b6e7-23b8-4126-9366-fa4a9f72fa8a","cluster":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:14:54.623Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"a620b995-97b0-4ad1-b08d-7fdc0fcb6bab","error":"certificate ray-head-cert-counter-8j287 is not ready at current generation 1"}
{"level":"error","ts":"2026-11-03T15:14:54.623Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"a620b995-97b0-4ad1-b08d-7fdc0fcb6bab","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-counter-8j287 is not ready at current generation 1","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:14:54.623Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"a620b995-97b0-4ad1-b08d-7fdc0fcb6bab","error":"mTLS secrets not ready: certificate ray-head-cert-counter-8j287 is not ready at current generation 1"}
{"level":"info","ts":"2026-11-03T15:14:54.623Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"a620b995-97b0-4ad1-b08d-7fdc0fcb6bab","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"counter-8j287","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:14:54.623Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"a620b995-97b0-4ad1-b08d-7fdc0fcb6bab","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"counter-8j287","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:14:54.623Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"a620b995-97b0-4ad1-b08d-7fdc0fcb6bab","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"counter-8j287","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:14:54.623Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"a620b995-97b0-4ad1-b08d-7fdc0fcb6bab"}
{"level":"error","ts":"2026-11-03T15:14:54.623Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"a620b995-97b0-4ad1-b08d-7fdc0fcb6bab","error":"mTLS secrets not ready: certificate ray-head-cert-counter-8j287 is not ready at current generation 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:14:54.943Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"a38dd1ff-0317-41f0-b181-0e0cba76b89f"}
{"level":"info","ts":"2026-11-03T15:14:54.943Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"a38dd1ff-0317-41f0-b181-0e0cba76b89f"}
{"level":"info","ts":"2026-11-03T15:14:54.944Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"a38dd1ff-0317-41f0-b181-0e0cba76b89f","1 head service found":"counter-8j287-head-svc"}
{"level":"info","ts":"2026-11-03T15:14:54.944Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"a38dd1ff-0317-41f0-b181-0e0cba76b89f"}
{"level":"info","ts":"2026-11-03T15:14:54.994Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"a38dd1ff-0317-41f0-b181-0e0cba76b89f","error":"certificate ray-head-cert-counter-8j287 is not ready at current generation 1"}
{"level":"error","ts":"2026-11-03T15:14:54.994Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"a38dd1ff-0317-41f0-b181-0e0cba76b89f","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-counter-8j287 is not ready at current generation 1","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:14:54.994Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"a38dd1ff-0317-41f0-b181-0e0cba76b89f","error":"mTLS secrets not ready: certificate ray-head-cert-counter-8j287 is not ready at current generation 1"}
{"level":"info","ts":"2026-11-03T15:14:54.994Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"a38dd1ff-0317-41f0-b181-0e0cba76b89f","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"counter-8j287","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:14:54.994Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"a38dd1ff-0317-41f0-b181-0e0cba76b89f","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"counter-8j287","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:14:54.994Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"a38dd1ff-0317-41f0-b181-0e0cba76b89f","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"counter-8j287","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:14:54.994Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"a38dd1ff-0317-41f0-b181-0e0cba76b89f"}
{"level":"error","ts":"2026-11-03T15:14:54.994Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"a38dd1ff-0317-41f0-b181-0e0cba76b89f","error":"mTLS secrets not ready: certificate ray-head-cert-counter-8j287 is not ready at current generation 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:14:55.073Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"2841b6e7-23b8-4126-9366-fa4a9f72fa8a","certificate":"ray-head-cert-counter-8j287"}
{"level":"info","ts":"2026-11-03T15:14:55.073Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"2841b6e7-23b8-4126-9366-fa4a9f72fa8a"}
{"level":"info","ts":"2026-11-03T15:14:55.635Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"acf1f449-dc5a-4134-8ab1-5c68bc57a959"}
{"level":"info","ts":"2026-11-03T15:14:55.635Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"acf1f449-dc5a-4134-8ab1-5c68bc57a959"}
{"level":"info","ts":"2026-11-03T15:14:55.635Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"acf1f449-dc5a-4134-8ab1-5c68bc57a959","1 head service found":"counter-8j287-head-svc"}
{"level":"info","ts":"2026-11-03T15:14:55.635Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"acf1f449-dc5a-4134-8ab1-5c68bc57a959"}
{"level":"info","ts":"2026-11-03T15:14:55.686Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"acf1f449-dc5a-4134-8ab1-5c68bc57a959","error":"certificate ray-head-cert-counter-8j287 is not ready at current generation 1"}
{"level":"error","ts":"2026-11-03T15:14:55.686Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"acf1f449-dc5a-4134-8ab1-5c68bc57a959","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-counter-8j287 is not ready at current generation 1","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:14:55.686Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"acf1f449-dc5a-4134-8ab1-5c68bc57a959","error":"mTLS secrets not ready: certificate ray-head-cert-counter-8j287 is not ready at current generation 1"}
{"level":"info","ts":"2026-11-03T15:14:55.686Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"acf1f449-dc5a-4134-8ab1-5c68bc57a959","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"counter-8j287","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:14:55.686Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"acf1f449-dc5a-4134-8ab1-5c68bc57a959","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"counter-8j287","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:14:55.686Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"acf1f449-dc5a-4134-8ab1-5c68bc57a959","filter labels":["test-ns-5lqjm",{"ray.io/cluster":"counter-8j287","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:14:55.686Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"acf1f449-dc5a-4134-8ab1-5c68bc57a959"}
{"level":"error","ts":"2026-11-03T15:14:55.686Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"acf1f449-dc5a-4134-8ab1-5c68bc57a959","error":"mTLS secrets not ready: certificate ray-head-cert-counter-8j287 is not ready at current generation 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:14:56.967Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"ed906f6b-259d-4cd8-a5b1-babccc7fc163"}
{"level":"info","ts":"2026-11-03T15:14:56.967Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"ed906f6b-259d-4cd8-a5b1-babccc7fc163"}
{"level":"info","ts":"2026-11-03T15:14:56.967Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"ed906f6b-259d-4cd8-a5b1-babccc7fc163","1 head service found":"counter-8j287-head-svc"}
{"level":"info","ts":"2026-11-03T15:14:56.967Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"ed906f6b-259d-4cd8-a5b1-babccc7fc163"}
{"level":"info","ts":"2026-11-03T15:14:57.067Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"ed906f6b-259d-4cd8-a5b1-babccc7fc163","mode":"IntegratedOAuth","cluster":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:14:57.067Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"ed906f6b-259d-4cd8-a5b1-babccc7fc163","cluster":"counter-8j287","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-11-03T15:14:57.067Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"ed906f6b-259d-4cd8-a5b1-babccc7fc163","cluster":"counter-8j287","authType":"IntegratedOAuth","serviceAccount":"counter-8j287-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-11-03T15:14:57.068Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"ed906f6b-259d-4cd8-a5b1-babccc7fc163","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"counter-8j287","ray.io/group":"headgroup","ray.io/identifier":"counter-8j287-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-11-03T15:14:57.068Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"ed906f6b-259d-4cd8-a5b1-babccc7fc163","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-11-03T15:14:57.068Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"ed906f6b-259d-4cd8-a5b1-babccc7fc163","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-11-03T15:14:57.068Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"ed906f6b-259d-4cd8-a5b1-babccc7fc163","rayNodeType":"head","generatedCmd":"ulimit -n ${RAY_START_ULIMIT_OPEN_FILES:-65536}; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-11-03T15:14:57.068Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"ed906f6b-259d-4cd8-a5b1-babccc7fc163","enabled":true}
{"level":"info","ts":"2026-11-03T15:14:57.183Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"ed906f6b-259d-4cd8-a5b1-babccc7fc163","name":"counter-8j287-head-zldss"}
{"level":"info","ts":"2026-11-03T15:14:57.183Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"ed906f6b-259d-4cd8-a5b1-babccc7fc163","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:14:57.183Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"ed906f6b-259d-4cd8-a5b1-babccc7fc163","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:14:57.183Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"ed906f6b-259d-4cd8-a5b1-babccc7fc163","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-11-03T15:14:57.183Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"ed906f6b-259d-4cd8-a5b1-babccc7fc163","group":"small-group","indices":{}}
{"level":"info","ts":"2026-11-03T15:14:57.183Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"ed906f6b-259d-4cd8-a5b1-babccc7fc163","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-11-03T15:14:57.284Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"ed906f6b-259d-4cd8-a5b1-babccc7fc163","creating worker for group":"small-group","index":0,"total":1,"replicaIndex":0}
{"level":"info","ts":"2026-11-03T15:14:57.284Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"ed906f6b-259d-4cd8-a5b1-babccc7fc163","nodeType":"worker","rayStartParams":{"address":"counter-8j287-head-svc.test-ns-5lqjm.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-11-03T15:14:57.284Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"ed906f6b-259d-4cd8-a5b1-babccc7fc163","rayStartCmd":"ray start  --address=counter-8j287-head-svc.test-ns-5lqjm.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-11-03T15:14:57.284Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"ed906f6b-259d-4cd8-a5b1-babccc7fc163","rayNodeType":"worker","generatedCmd":"ulimit -n ${RAY_START_ULIMIT_OPEN_FILES:-65536}; ray start  --address=counter-8j287-head-svc.test-ns-5lqjm.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-11-03T15:14:57.284Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"ed906f6b-259d-4cd8-a5b1-babccc7fc163","enabled":true}
{"level":"info","ts":"2026-11-03T15:14:57.345Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"ed906f6b-259d-4cd8-a5b1-babccc7fc163","name":"counter-8j287-small-group-worker-j9tg5"}
{"level":"info","ts":"2026-11-03T15:14:57.345Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"ed906f6b-259d-4cd8-a5b1-babccc7fc163","podName":"counter-8j287-head-zldss","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:14:57.345Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"ed906f6b-259d-4cd8-a5b1-babccc7fc163","name":"counter-8j287","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:57Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:14:57.405Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"e9d30f12-26de-477a-ad92-e0e2a169e2d7"}
{"level":"info","ts":"2026-11-03T15:14:57.405Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"e9d30f12-26de-477a-ad92-e0e2a169e2d7"}
{"level":"info","ts":"2026-11-03T15:14:57.405Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"e9d30f12-26de-477a-ad92-e0e2a169e2d7","1 head service found":"counter-8j287-head-svc"}
{"level":"info","ts":"2026-11-03T15:14:57.405Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"e9d30f12-26de-477a-ad92-e0e2a169e2d7","Found 1 head Pod":"counter-8j287-head-zldss","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:14:57.405Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"e9d30f12-26de-477a-ad92-e0e2a169e2d7","head Pod":"counter-8j287-head-zldss","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-8j287-head-zldss. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:14:57.405Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"e9d30f12-26de-477a-ad92-e0e2a169e2d7","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:14:57.405Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"e9d30f12-26de-477a-ad92-e0e2a169e2d7","worker Pod":"counter-8j287-small-group-worker-j9tg5","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-8j287-small-group-worker-j9tg5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:14:57.405Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"e9d30f12-26de-477a-ad92-e0e2a169e2d7","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:14:57.405Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"e9d30f12-26de-477a-ad92-e0e2a169e2d7","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:14:57.405Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"e9d30f12-26de-477a-ad92-e0e2a169e2d7","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:14:57.405Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"e9d30f12-26de-477a-ad92-e0e2a169e2d7","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:14:57.405Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"e9d30f12-26de-477a-ad92-e0e2a169e2d7","podName":"counter-8j287-head-zldss","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:14:57.405Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"e9d30f12-26de-477a-ad92-e0e2a169e2d7","podName":"counter-8j287-head-zldss","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:14:57.405Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"24cfde94-1d64-4ede-a04a-e8ac3724b9ea","cluster":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:14:57.405Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"e9d30f12-26de-477a-ad92-e0e2a169e2d7","name":"counter-8j287","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:57Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:14:57.406Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"aeee1fa5-298a-4e13-b5c9-b7bd4ff08a60","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:14:57.406Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"aeee1fa5-298a-4e13-b5c9-b7bd4ff08a60","name":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:14:57.406Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"aeee1fa5-298a-4e13-b5c9-b7bd4ff08a60","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:14:57.406Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"aeee1fa5-298a-4e13-b5c9-b7bd4ff08a60"}
{"level":"info","ts":"2026-11-03T15:14:57.406Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"aeee1fa5-298a-4e13-b5c9-b7bd4ff08a60","RayCluster":"counter-8j287","State":""}
{"level":"info","ts":"2026-11-03T15:14:57.406Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"aeee1fa5-298a-4e13-b5c9-b7bd4ff08a60","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:57Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:14:57.406Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"aeee1fa5-298a-4e13-b5c9-b7bd4ff08a60","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:14:57.460Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"e9d30f12-26de-477a-ad92-e0e2a169e2d7","name":"counter-8j287","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-8j287\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-5lqjm","name":"counter-8j287"}}
{"level":"info","ts":"2026-11-03T15:14:57.460Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"e9d30f12-26de-477a-ad92-e0e2a169e2d7"}
{"level":"error","ts":"2026-11-03T15:14:57.460Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"e9d30f12-26de-477a-ad92-e0e2a169e2d7","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-8j287\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:14:57.460Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"5fd7a43a-3972-4a90-ad6a-c597b9938399"}
{"level":"info","ts":"2026-11-03T15:14:57.460Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"5fd7a43a-3972-4a90-ad6a-c597b9938399"}
{"level":"info","ts":"2026-11-03T15:14:57.460Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"5fd7a43a-3972-4a90-ad6a-c597b9938399","1 head service found":"counter-8j287-head-svc"}
{"level":"info","ts":"2026-11-03T15:14:57.460Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"5fd7a43a-3972-4a90-ad6a-c597b9938399","Found 1 head Pod":"counter-8j287-head-zldss","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:14:57.460Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"5fd7a43a-3972-4a90-ad6a-c597b9938399","head Pod":"counter-8j287-head-zldss","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-8j287-head-zldss. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:14:57.460Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"5fd7a43a-3972-4a90-ad6a-c597b9938399","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:14:57.460Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"5fd7a43a-3972-4a90-ad6a-c597b9938399","worker Pod":"counter-8j287-small-group-worker-j9tg5","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-8j287-small-group-worker-j9tg5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:14:57.461Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"5fd7a43a-3972-4a90-ad6a-c597b9938399","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:14:57.461Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"5fd7a43a-3972-4a90-ad6a-c597b9938399","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:14:57.461Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"5fd7a43a-3972-4a90-ad6a-c597b9938399","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:14:57.461Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"5fd7a43a-3972-4a90-ad6a-c597b9938399","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:14:57.461Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"5fd7a43a-3972-4a90-ad6a-c597b9938399","podName":"counter-8j287-head-zldss","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:14:57.461Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"5fd7a43a-3972-4a90-ad6a-c597b9938399","podName":"counter-8j287-head-zldss","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:14:57.461Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"5fd7a43a-3972-4a90-ad6a-c597b9938399","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:14:57.461Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"5fd7a43a-3972-4a90-ad6a-c597b9938399","seconds":300}
{"level":"info","ts":"2026-11-03T15:14:57.466Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"e5b838a0-7648-402b-bdfb-876c3acf2741","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:14:57.466Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"e5b838a0-7648-402b-bdfb-876c3acf2741","name":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:14:57.467Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"e5b838a0-7648-402b-bdfb-876c3acf2741","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:14:57.467Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"e5b838a0-7648-402b-bdfb-876c3acf2741"}
{"level":"info","ts":"2026-11-03T15:14:57.467Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"e5b838a0-7648-402b-bdfb-876c3acf2741","RayCluster":"counter-8j287","State":""}
{"level":"info","ts":"2026-11-03T15:14:57.467Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"e5b838a0-7648-402b-bdfb-876c3acf2741","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:57Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:57Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:14:57.468Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"9aa295e3-7b41-4763-9fa9-27be1ac41a14"}
{"level":"info","ts":"2026-11-03T15:14:57.468Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"9aa295e3-7b41-4763-9fa9-27be1ac41a14"}
{"level":"info","ts":"2026-11-03T15:14:57.468Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"9aa295e3-7b41-4763-9fa9-27be1ac41a14","1 head service found":"counter-8j287-head-svc"}
{"level":"info","ts":"2026-11-03T15:14:57.468Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"9aa295e3-7b41-4763-9fa9-27be1ac41a14","Found 1 head Pod":"counter-8j287-head-zldss","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:14:57.468Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"9aa295e3-7b41-4763-9fa9-27be1ac41a14","head Pod":"counter-8j287-head-zldss","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-8j287-head-zldss. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:14:57.468Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"9aa295e3-7b41-4763-9fa9-27be1ac41a14","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:14:57.468Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"9aa295e3-7b41-4763-9fa9-27be1ac41a14","worker Pod":"counter-8j287-small-group-worker-j9tg5","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-8j287-small-group-worker-j9tg5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:14:57.468Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"9aa295e3-7b41-4763-9fa9-27be1ac41a14","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:14:57.468Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"9aa295e3-7b41-4763-9fa9-27be1ac41a14","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:14:57.468Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"9aa295e3-7b41-4763-9fa9-27be1ac41a14","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:14:57.468Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"9aa295e3-7b41-4763-9fa9-27be1ac41a14","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:14:57.468Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"9aa295e3-7b41-4763-9fa9-27be1ac41a14","podName":"counter-8j287-head-zldss","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:14:57.468Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"9aa295e3-7b41-4763-9fa9-27be1ac41a14","podName":"counter-8j287-head-zldss","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:14:57.468Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"9aa295e3-7b41-4763-9fa9-27be1ac41a14","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:14:57.468Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"9aa295e3-7b41-4763-9fa9-27be1ac41a14","seconds":300}
{"level":"info","ts":"2026-11-03T15:14:57.879Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"72c581e7-a914-4976-9b22-1b947a7293bf"}
{"level":"info","ts":"2026-11-03T15:14:57.879Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"72c581e7-a914-4976-9b22-1b947a7293bf"}
{"level":"info","ts":"2026-11-03T15:14:57.879Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"72c581e7-a914-4976-9b22-1b947a7293bf","1 head service found":"counter-8j287-head-svc"}
{"level":"info","ts":"2026-11-03T15:14:57.879Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"72c581e7-a914-4976-9b22-1b947a7293bf","Found 1 head Pod":"counter-8j287-head-zldss","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:14:57.879Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"72c581e7-a914-4976-9b22-1b947a7293bf","head Pod":"counter-8j287-head-zldss","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-8j287-head-zldss. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:14:57.879Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"72c581e7-a914-4976-9b22-1b947a7293bf","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:14:57.879Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"72c581e7-a914-4976-9b22-1b947a7293bf","worker Pod":"counter-8j287-small-group-worker-j9tg5","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-8j287-small-group-worker-j9tg5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:14:57.879Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"72c581e7-a914-4976-9b22-1b947a7293bf","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:14:57.879Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"72c581e7-a914-4976-9b22-1b947a7293bf","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:14:57.879Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"72c581e7-a914-4976-9b22-1b947a7293bf","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:14:57.879Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"72c581e7-a914-4976-9b22-1b947a7293bf","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:14:57.879Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"72c581e7-a914-4976-9b22-1b947a7293bf","podName":"counter-8j287-head-zldss","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:14:57.879Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"72c581e7-a914-4976-9b22-1b947a7293bf","podName":"counter-8j287-head-zldss","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:14:57.879Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"72c581e7-a914-4976-9b22-1b947a7293bf","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:14:57.879Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"72c581e7-a914-4976-9b22-1b947a7293bf","seconds":300}
{"level":"info","ts":"2026-11-03T15:14:58.094Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"490981da-9d69-4e51-ae91-dbb9bdf60a3f"}
{"level":"info","ts":"2026-11-03T15:14:58.094Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"490981da-9d69-4e51-ae91-dbb9bdf60a3f"}
{"level":"info","ts":"2026-11-03T15:14:58.094Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"490981da-9d69-4e51-ae91-dbb9bdf60a3f","1 head service found":"counter-8j287-head-svc"}
{"level":"info","ts":"2026-11-03T15:14:58.094Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"490981da-9d69-4e51-ae91-dbb9bdf60a3f","Found 1 head Pod":"counter-8j287-head-zldss","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:14:58.094Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"490981da-9d69-4e51-ae91-dbb9bdf60a3f","head Pod":"counter-8j287-head-zldss","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-8j287-head-zldss. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:14:58.094Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"490981da-9d69-4e51-ae91-dbb9bdf60a3f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:14:58.094Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"490981da-9d69-4e51-ae91-dbb9bdf60a3f","worker Pod":"counter-8j287-small-group-worker-j9tg5","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-8j287-small-group-worker-j9tg5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:14:58.094Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"490981da-9d69-4e51-ae91-dbb9bdf60a3f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:14:58.094Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"490981da-9d69-4e51-ae91-dbb9bdf60a3f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:14:58.094Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"490981da-9d69-4e51-ae91-dbb9bdf60a3f","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:14:58.094Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"490981da-9d69-4e51-ae91-dbb9bdf60a3f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:14:58.095Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"490981da-9d69-4e51-ae91-dbb9bdf60a3f","podName":"counter-8j287-head-zldss","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:14:58.095Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"490981da-9d69-4e51-ae91-dbb9bdf60a3f","podName":"counter-8j287-head-zldss","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:14:58.095Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"490981da-9d69-4e51-ae91-dbb9bdf60a3f","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:14:58.095Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"490981da-9d69-4e51-ae91-dbb9bdf60a3f","seconds":300}
{"level":"info","ts":"2026-11-03T15:14:58.718Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"759243d2-86ff-46a9-8458-bc4124c3c7f7"}
{"level":"info","ts":"2026-11-03T15:14:58.718Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"759243d2-86ff-46a9-8458-bc4124c3c7f7"}
{"level":"info","ts":"2026-11-03T15:14:58.718Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"759243d2-86ff-46a9-8458-bc4124c3c7f7","1 head service found":"counter-8j287-head-svc"}
{"level":"info","ts":"2026-11-03T15:14:58.718Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"759243d2-86ff-46a9-8458-bc4124c3c7f7","Found 1 head Pod":"counter-8j287-head-zldss","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:14:58.718Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"759243d2-86ff-46a9-8458-bc4124c3c7f7","head Pod":"counter-8j287-head-zldss","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-8j287-head-zldss. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:14:58.718Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"759243d2-86ff-46a9-8458-bc4124c3c7f7","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:14:58.718Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"759243d2-86ff-46a9-8458-bc4124c3c7f7","worker Pod":"counter-8j287-small-group-worker-j9tg5","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-8j287-small-group-worker-j9tg5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:14:58.718Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"759243d2-86ff-46a9-8458-bc4124c3c7f7","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:14:58.718Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"759243d2-86ff-46a9-8458-bc4124c3c7f7","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:14:58.718Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"759243d2-86ff-46a9-8458-bc4124c3c7f7","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:14:58.718Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"759243d2-86ff-46a9-8458-bc4124c3c7f7","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:14:58.718Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"759243d2-86ff-46a9-8458-bc4124c3c7f7","podName":"counter-8j287-small-group-worker-j9tg5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:14:58.718Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"759243d2-86ff-46a9-8458-bc4124c3c7f7","podName":"counter-8j287-small-group-worker-j9tg5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:14:58.718Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"759243d2-86ff-46a9-8458-bc4124c3c7f7","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:14:58.718Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"759243d2-86ff-46a9-8458-bc4124c3c7f7","seconds":300}
{"level":"info","ts":"2026-11-03T15:14:58.876Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"c54a028d-40a1-4c0f-adc4-209bad0e1288"}
{"level":"info","ts":"2026-11-03T15:14:58.876Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"c54a028d-40a1-4c0f-adc4-209bad0e1288"}
{"level":"info","ts":"2026-11-03T15:14:58.876Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"c54a028d-40a1-4c0f-adc4-209bad0e1288","1 head service found":"counter-8j287-head-svc"}
{"level":"info","ts":"2026-11-03T15:14:58.876Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"c54a028d-40a1-4c0f-adc4-209bad0e1288","Found 1 head Pod":"counter-8j287-head-zldss","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:14:58.876Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"c54a028d-40a1-4c0f-adc4-209bad0e1288","head Pod":"counter-8j287-head-zldss","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-8j287-head-zldss. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:14:58.876Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"c54a028d-40a1-4c0f-adc4-209bad0e1288","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:14:58.876Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"c54a028d-40a1-4c0f-adc4-209bad0e1288","worker Pod":"counter-8j287-small-group-worker-j9tg5","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-8j287-small-group-worker-j9tg5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:14:58.876Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"c54a028d-40a1-4c0f-adc4-209bad0e1288","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:14:58.876Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"c54a028d-40a1-4c0f-adc4-209bad0e1288","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:14:58.876Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"c54a028d-40a1-4c0f-adc4-209bad0e1288","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:14:58.876Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"c54a028d-40a1-4c0f-adc4-209bad0e1288","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:14:58.876Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"c54a028d-40a1-4c0f-adc4-209bad0e1288","podName":"counter-8j287-head-zldss","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:14:58.876Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"c54a028d-40a1-4c0f-adc4-209bad0e1288","podName":"counter-8j287-head-zldss","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:14:58.876Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"c54a028d-40a1-4c0f-adc4-209bad0e1288","name":"counter-8j287","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:57Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:14:58.938Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"83735fb9-9459-4cb6-aebb-754dc20e5e14","cluster":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:14:58.938Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"8791d067-d806-4307-80f2-8c9bd66f865d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:14:58.938Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"8791d067-d806-4307-80f2-8c9bd66f865d","name":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:14:58.938Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"8791d067-d806-4307-80f2-8c9bd66f865d","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:14:58.938Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"8791d067-d806-4307-80f2-8c9bd66f865d"}
{"level":"info","ts":"2026-11-03T15:14:58.938Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"8791d067-d806-4307-80f2-8c9bd66f865d","RayCluster":"counter-8j287","State":""}
{"level":"info","ts":"2026-11-03T15:14:58.938Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"8791d067-d806-4307-80f2-8c9bd66f865d","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:57Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:14:58.938Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"8791d067-d806-4307-80f2-8c9bd66f865d","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:14:58.999Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"ecc98e31-7282-4e29-a740-f939e6359c71","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:14:58.999Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"ecc98e31-7282-4e29-a740-f939e6359c71","name":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:14:58.999Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"ecc98e31-7282-4e29-a740-f939e6359c71","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:14:58.999Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"ecc98e31-7282-4e29-a740-f939e6359c71"}
{"level":"info","ts":"2026-11-03T15:14:58.999Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"ecc98e31-7282-4e29-a740-f939e6359c71","RayCluster":"counter-8j287","State":""}
{"level":"info","ts":"2026-11-03T15:14:58.999Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"ecc98e31-7282-4e29-a740-f939e6359c71","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:14:59.242Z","logger":"controllers.RayClusterMTLS","msg":"Updating head certificate","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"83735fb9-9459-4cb6-aebb-754dc20e5e14","certificate":"ray-head-cert-counter-8j287","ipAddresses":["10.132.0.37","127.0.0.1"]}
{"level":"info","ts":"2026-11-03T15:14:59.496Z","logger":"controllers.RayClusterMTLS","msg":"Updating worker certificate","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"83735fb9-9459-4cb6-aebb-754dc20e5e14","certificate":"ray-worker-cert-counter-8j287","ipAddresses":["10.134.0.35","127.0.0.1"]}
{"level":"info","ts":"2026-11-03T15:14:59.750Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"83735fb9-9459-4cb6-aebb-754dc20e5e14","certificate":"ray-head-cert-counter-8j287"}
{"level":"info","ts":"2026-11-03T15:14:59.750Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"83735fb9-9459-4cb6-aebb-754dc20e5e14"}
{"level":"info","ts":"2026-11-03T15:15:00.938Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"bf1a92fb-6ca8-4dbf-8924-e41e985607e9"}
{"level":"info","ts":"2026-11-03T15:15:00.938Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"bf1a92fb-6ca8-4dbf-8924-e41e985607e9"}
{"level":"info","ts":"2026-11-03T15:15:00.938Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"bf1a92fb-6ca8-4dbf-8924-e41e985607e9","1 head service found":"counter-8j287-head-svc"}
{"level":"info","ts":"2026-11-03T15:15:00.938Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"bf1a92fb-6ca8-4dbf-8924-e41e985607e9","Found 1 head Pod":"counter-8j287-head-zldss","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:15:00.938Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"bf1a92fb-6ca8-4dbf-8924-e41e985607e9","head Pod":"counter-8j287-head-zldss","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-8j287-head-zldss. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:15:00.938Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"bf1a92fb-6ca8-4dbf-8924-e41e985607e9","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:15:00.938Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"bf1a92fb-6ca8-4dbf-8924-e41e985607e9","worker Pod":"counter-8j287-small-group-worker-j9tg5","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-8j287-small-group-worker-j9tg5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:15:00.938Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"bf1a92fb-6ca8-4dbf-8924-e41e985607e9","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:15:00.938Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"bf1a92fb-6ca8-4dbf-8924-e41e985607e9","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:15:00.938Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"bf1a92fb-6ca8-4dbf-8924-e41e985607e9","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:15:00.938Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"bf1a92fb-6ca8-4dbf-8924-e41e985607e9","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:15:00.938Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"bf1a92fb-6ca8-4dbf-8924-e41e985607e9","podName":"counter-8j287-small-group-worker-j9tg5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:15:00.938Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"bf1a92fb-6ca8-4dbf-8924-e41e985607e9","podName":"counter-8j287-small-group-worker-j9tg5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:15:00.939Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"bf1a92fb-6ca8-4dbf-8924-e41e985607e9","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:15:00.939Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"bf1a92fb-6ca8-4dbf-8924-e41e985607e9","seconds":300}
{"level":"info","ts":"2026-11-03T15:15:02.000Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"6b9d3bf0-b5a0-4677-95fe-6ea8af552fad","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:15:02.000Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"6b9d3bf0-b5a0-4677-95fe-6ea8af552fad","name":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:15:02.000Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"6b9d3bf0-b5a0-4677-95fe-6ea8af552fad","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:15:02.000Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"6b9d3bf0-b5a0-4677-95fe-6ea8af552fad"}
{"level":"info","ts":"2026-11-03T15:15:02.000Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"6b9d3bf0-b5a0-4677-95fe-6ea8af552fad","RayCluster":"counter-8j287","State":""}
{"level":"info","ts":"2026-11-03T15:15:02.000Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"6b9d3bf0-b5a0-4677-95fe-6ea8af552fad","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:15:05.001Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"0d7c49da-0732-418b-bb97-ba9c16038d71","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:15:05.001Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"0d7c49da-0732-418b-bb97-ba9c16038d71","name":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:15:05.001Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"0d7c49da-0732-418b-bb97-ba9c16038d71","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:15:05.001Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"0d7c49da-0732-418b-bb97-ba9c16038d71"}
{"level":"info","ts":"2026-11-03T15:15:05.001Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"0d7c49da-0732-418b-bb97-ba9c16038d71","RayCluster":"counter-8j287","State":""}
{"level":"info","ts":"2026-11-03T15:15:05.001Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"0d7c49da-0732-418b-bb97-ba9c16038d71","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:15:08.003Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"48de03c8-4f92-4d91-beed-eaadfe336547","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:15:08.003Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"48de03c8-4f92-4d91-beed-eaadfe336547","name":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:15:08.003Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"48de03c8-4f92-4d91-beed-eaadfe336547","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:15:08.003Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"48de03c8-4f92-4d91-beed-eaadfe336547"}
{"level":"info","ts":"2026-11-03T15:15:08.003Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"48de03c8-4f92-4d91-beed-eaadfe336547","RayCluster":"counter-8j287","State":""}
{"level":"info","ts":"2026-11-03T15:15:08.003Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"48de03c8-4f92-4d91-beed-eaadfe336547","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:15:11.004Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"866863f5-ac54-48f7-8e86-2df5d38febd9","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:15:11.004Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"866863f5-ac54-48f7-8e86-2df5d38febd9","name":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:15:11.004Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"866863f5-ac54-48f7-8e86-2df5d38febd9","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:15:11.004Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"866863f5-ac54-48f7-8e86-2df5d38febd9"}
{"level":"info","ts":"2026-11-03T15:15:11.004Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"866863f5-ac54-48f7-8e86-2df5d38febd9","RayCluster":"counter-8j287","State":""}
{"level":"info","ts":"2026-11-03T15:15:11.004Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"866863f5-ac54-48f7-8e86-2df5d38febd9","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:15:14.005Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"da22c027-f493-409e-a0ef-3510408c7719","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:15:14.005Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"da22c027-f493-409e-a0ef-3510408c7719","name":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:15:14.005Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"da22c027-f493-409e-a0ef-3510408c7719","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:15:14.005Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"da22c027-f493-409e-a0ef-3510408c7719"}
{"level":"info","ts":"2026-11-03T15:15:14.005Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"da22c027-f493-409e-a0ef-3510408c7719","RayCluster":"counter-8j287","State":""}
{"level":"info","ts":"2026-11-03T15:15:14.005Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"da22c027-f493-409e-a0ef-3510408c7719","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:15:17.006Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"e0df7d02-c62d-4776-9799-48907893e89f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:15:17.006Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"e0df7d02-c62d-4776-9799-48907893e89f","name":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:15:17.006Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"e0df7d02-c62d-4776-9799-48907893e89f","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:15:17.006Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"e0df7d02-c62d-4776-9799-48907893e89f"}
{"level":"info","ts":"2026-11-03T15:15:17.006Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"e0df7d02-c62d-4776-9799-48907893e89f","RayCluster":"counter-8j287","State":""}
{"level":"info","ts":"2026-11-03T15:15:17.006Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"e0df7d02-c62d-4776-9799-48907893e89f","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:15:20.007Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"600358e9-e7c5-4cba-a15d-a99acd14909b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:15:20.007Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"600358e9-e7c5-4cba-a15d-a99acd14909b","name":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:15:20.007Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"600358e9-e7c5-4cba-a15d-a99acd14909b","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:15:20.007Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"600358e9-e7c5-4cba-a15d-a99acd14909b"}
{"level":"info","ts":"2026-11-03T15:15:20.007Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"600358e9-e7c5-4cba-a15d-a99acd14909b","RayCluster":"counter-8j287","State":""}
{"level":"info","ts":"2026-11-03T15:15:20.007Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"600358e9-e7c5-4cba-a15d-a99acd14909b","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:15:23.008Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"da3d650f-7096-4258-9788-b472013088b1","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:15:23.008Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"da3d650f-7096-4258-9788-b472013088b1","name":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:15:23.008Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"da3d650f-7096-4258-9788-b472013088b1","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:15:23.008Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"da3d650f-7096-4258-9788-b472013088b1"}
{"level":"info","ts":"2026-11-03T15:15:23.008Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"da3d650f-7096-4258-9788-b472013088b1","RayCluster":"counter-8j287","State":""}
{"level":"info","ts":"2026-11-03T15:15:23.008Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"da3d650f-7096-4258-9788-b472013088b1","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:15:26.009Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"b94617da-a0c9-459a-950e-29a1162e09ef","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:15:26.009Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"b94617da-a0c9-459a-950e-29a1162e09ef","name":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:15:26.009Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"b94617da-a0c9-459a-950e-29a1162e09ef","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:15:26.009Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"b94617da-a0c9-459a-950e-29a1162e09ef"}
{"level":"info","ts":"2026-11-03T15:15:26.009Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"b94617da-a0c9-459a-950e-29a1162e09ef","RayCluster":"counter-8j287","State":""}
{"level":"info","ts":"2026-11-03T15:15:26.009Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"b94617da-a0c9-459a-950e-29a1162e09ef","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:15:29.010Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"ec82411a-1286-4270-b2bc-37936fad1da0","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:15:29.010Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"ec82411a-1286-4270-b2bc-37936fad1da0","name":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:15:29.010Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"ec82411a-1286-4270-b2bc-37936fad1da0","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:15:29.010Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"ec82411a-1286-4270-b2bc-37936fad1da0"}
{"level":"info","ts":"2026-11-03T15:15:29.010Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"ec82411a-1286-4270-b2bc-37936fad1da0","RayCluster":"counter-8j287","State":""}
{"level":"info","ts":"2026-11-03T15:15:29.010Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"ec82411a-1286-4270-b2bc-37936fad1da0","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:15:29.751Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"77b9f736-1323-4b7c-92f1-1ef2719b58a9","cluster":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:15:32.011Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"8fae0ab5-062e-46dd-9863-1b0c378be1f9","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:15:32.011Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"8fae0ab5-062e-46dd-9863-1b0c378be1f9","name":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:15:32.011Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"8fae0ab5-062e-46dd-9863-1b0c378be1f9","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:15:32.011Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"8fae0ab5-062e-46dd-9863-1b0c378be1f9"}
{"level":"info","ts":"2026-11-03T15:15:32.011Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"8fae0ab5-062e-46dd-9863-1b0c378be1f9","RayCluster":"counter-8j287","State":""}
{"level":"info","ts":"2026-11-03T15:15:32.011Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"8fae0ab5-062e-46dd-9863-1b0c378be1f9","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:15:35.012Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"eb3bf470-1923-4500-8506-eea56a1044d0","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:15:35.012Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"eb3bf470-1923-4500-8506-eea56a1044d0","name":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:15:35.012Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"eb3bf470-1923-4500-8506-eea56a1044d0","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:15:35.012Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"eb3bf470-1923-4500-8506-eea56a1044d0"}
{"level":"info","ts":"2026-11-03T15:15:35.012Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"eb3bf470-1923-4500-8506-eea56a1044d0","RayCluster":"counter-8j287","State":""}
{"level":"info","ts":"2026-11-03T15:15:35.012Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"eb3bf470-1923-4500-8506-eea56a1044d0","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:15:38.014Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"fedf4690-7da7-4ba5-98b9-4c143365b944","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:15:38.014Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"fedf4690-7da7-4ba5-98b9-4c143365b944","name":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:15:38.014Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"fedf4690-7da7-4ba5-98b9-4c143365b944","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:15:38.014Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"fedf4690-7da7-4ba5-98b9-4c143365b944"}
{"level":"info","ts":"2026-11-03T15:15:38.014Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"fedf4690-7da7-4ba5-98b9-4c143365b944","RayCluster":"counter-8j287","State":""}
{"level":"info","ts":"2026-11-03T15:15:38.014Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"fedf4690-7da7-4ba5-98b9-4c143365b944","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:15:41.014Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"197adf25-e871-4ad0-8b40-a76200864d14","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:15:41.014Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"197adf25-e871-4ad0-8b40-a76200864d14","name":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:15:41.014Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"197adf25-e871-4ad0-8b40-a76200864d14","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:15:41.015Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"197adf25-e871-4ad0-8b40-a76200864d14"}
{"level":"info","ts":"2026-11-03T15:15:41.015Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"197adf25-e871-4ad0-8b40-a76200864d14","RayCluster":"counter-8j287","State":""}
{"level":"info","ts":"2026-11-03T15:15:41.015Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"197adf25-e871-4ad0-8b40-a76200864d14","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:15:44.015Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"2519289e-d7ca-40aa-bd95-1b0e5b87a86a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:15:44.015Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"2519289e-d7ca-40aa-bd95-1b0e5b87a86a","name":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:15:44.015Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"2519289e-d7ca-40aa-bd95-1b0e5b87a86a","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:15:44.015Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"2519289e-d7ca-40aa-bd95-1b0e5b87a86a"}
{"level":"info","ts":"2026-11-03T15:15:44.016Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"2519289e-d7ca-40aa-bd95-1b0e5b87a86a","RayCluster":"counter-8j287","State":""}
{"level":"info","ts":"2026-11-03T15:15:44.016Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"2519289e-d7ca-40aa-bd95-1b0e5b87a86a","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:15:47.016Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"470e5ad4-0622-404e-b610-3583d9f119ab","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:15:47.016Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"470e5ad4-0622-404e-b610-3583d9f119ab","name":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:15:47.016Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"470e5ad4-0622-404e-b610-3583d9f119ab","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:15:47.017Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"470e5ad4-0622-404e-b610-3583d9f119ab"}
{"level":"info","ts":"2026-11-03T15:15:47.017Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"470e5ad4-0622-404e-b610-3583d9f119ab","RayCluster":"counter-8j287","State":""}
{"level":"info","ts":"2026-11-03T15:15:47.017Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"470e5ad4-0622-404e-b610-3583d9f119ab","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:15:50.017Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"2a4fac75-a777-4536-9bef-9fe9d03b338f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:15:50.017Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"2a4fac75-a777-4536-9bef-9fe9d03b338f","name":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:15:50.017Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"2a4fac75-a777-4536-9bef-9fe9d03b338f","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:15:50.018Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"2a4fac75-a777-4536-9bef-9fe9d03b338f"}
{"level":"info","ts":"2026-11-03T15:15:50.018Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"2a4fac75-a777-4536-9bef-9fe9d03b338f","RayCluster":"counter-8j287","State":""}
{"level":"info","ts":"2026-11-03T15:15:50.018Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"2a4fac75-a777-4536-9bef-9fe9d03b338f","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:15:53.019Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"3a59baef-6dfc-4496-8a4f-4dd31c1237f5","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:15:53.019Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"3a59baef-6dfc-4496-8a4f-4dd31c1237f5","name":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:15:53.019Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"3a59baef-6dfc-4496-8a4f-4dd31c1237f5","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:15:53.019Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"3a59baef-6dfc-4496-8a4f-4dd31c1237f5"}
{"level":"info","ts":"2026-11-03T15:15:53.019Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"3a59baef-6dfc-4496-8a4f-4dd31c1237f5","RayCluster":"counter-8j287","State":""}
{"level":"info","ts":"2026-11-03T15:15:53.019Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"3a59baef-6dfc-4496-8a4f-4dd31c1237f5","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:15:56.020Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"88fc46c6-245d-4534-b9c9-357a2e9f1679","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:15:56.021Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"88fc46c6-245d-4534-b9c9-357a2e9f1679","name":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:15:56.021Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"88fc46c6-245d-4534-b9c9-357a2e9f1679","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:15:56.021Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"88fc46c6-245d-4534-b9c9-357a2e9f1679"}
{"level":"info","ts":"2026-11-03T15:15:56.021Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"88fc46c6-245d-4534-b9c9-357a2e9f1679","RayCluster":"counter-8j287","State":""}
{"level":"info","ts":"2026-11-03T15:15:56.021Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"88fc46c6-245d-4534-b9c9-357a2e9f1679","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:15:59.022Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"0c6d02b7-582c-45d1-8286-15cd4e7e845f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:15:59.022Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"0c6d02b7-582c-45d1-8286-15cd4e7e845f","name":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:15:59.022Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"0c6d02b7-582c-45d1-8286-15cd4e7e845f","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:15:59.022Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"0c6d02b7-582c-45d1-8286-15cd4e7e845f"}
{"level":"info","ts":"2026-11-03T15:15:59.022Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"0c6d02b7-582c-45d1-8286-15cd4e7e845f","RayCluster":"counter-8j287","State":""}
{"level":"info","ts":"2026-11-03T15:15:59.022Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"0c6d02b7-582c-45d1-8286-15cd4e7e845f","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:16:02.023Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"78f26b9d-0224-4c4d-81b4-2d28d973ee78","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:16:02.023Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"78f26b9d-0224-4c4d-81b4-2d28d973ee78","name":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:16:02.023Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"78f26b9d-0224-4c4d-81b4-2d28d973ee78","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:16:02.024Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"78f26b9d-0224-4c4d-81b4-2d28d973ee78"}
{"level":"info","ts":"2026-11-03T15:16:02.024Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"78f26b9d-0224-4c4d-81b4-2d28d973ee78","RayCluster":"counter-8j287","State":""}
{"level":"info","ts":"2026-11-03T15:16:02.024Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"78f26b9d-0224-4c4d-81b4-2d28d973ee78","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:16:05.024Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"1b1d75ba-65e3-4171-9a33-1c4a2885f807","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:16:05.024Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"1b1d75ba-65e3-4171-9a33-1c4a2885f807","name":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:16:05.024Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"1b1d75ba-65e3-4171-9a33-1c4a2885f807","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:16:05.024Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"1b1d75ba-65e3-4171-9a33-1c4a2885f807"}
{"level":"info","ts":"2026-11-03T15:16:05.024Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"1b1d75ba-65e3-4171-9a33-1c4a2885f807","RayCluster":"counter-8j287","State":""}
{"level":"info","ts":"2026-11-03T15:16:05.024Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"1b1d75ba-65e3-4171-9a33-1c4a2885f807","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:16:08.025Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"e37ded56-462e-430a-8ad8-ecaaee1db271","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:16:08.025Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"e37ded56-462e-430a-8ad8-ecaaee1db271","name":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:16:08.025Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"e37ded56-462e-430a-8ad8-ecaaee1db271","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:16:08.026Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"e37ded56-462e-430a-8ad8-ecaaee1db271"}
{"level":"info","ts":"2026-11-03T15:16:08.026Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"e37ded56-462e-430a-8ad8-ecaaee1db271","RayCluster":"counter-8j287","State":""}
{"level":"info","ts":"2026-11-03T15:16:08.026Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"e37ded56-462e-430a-8ad8-ecaaee1db271","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:16:11.026Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"02db715e-99a8-494e-a1ce-928c5519d7c5","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:16:11.026Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"02db715e-99a8-494e-a1ce-928c5519d7c5","name":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:16:11.026Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"02db715e-99a8-494e-a1ce-928c5519d7c5","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:16:11.027Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"02db715e-99a8-494e-a1ce-928c5519d7c5"}
{"level":"info","ts":"2026-11-03T15:16:11.027Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"02db715e-99a8-494e-a1ce-928c5519d7c5","RayCluster":"counter-8j287","State":""}
{"level":"info","ts":"2026-11-03T15:16:11.027Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"02db715e-99a8-494e-a1ce-928c5519d7c5","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:16:14.027Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"b5f1c303-03c4-4611-b8af-e0c0249ce734","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:16:14.027Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"b5f1c303-03c4-4611-b8af-e0c0249ce734","name":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:16:14.027Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"b5f1c303-03c4-4611-b8af-e0c0249ce734","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:16:14.027Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"b5f1c303-03c4-4611-b8af-e0c0249ce734"}
{"level":"info","ts":"2026-11-03T15:16:14.027Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"b5f1c303-03c4-4611-b8af-e0c0249ce734","RayCluster":"counter-8j287","State":""}
{"level":"info","ts":"2026-11-03T15:16:14.027Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"b5f1c303-03c4-4611-b8af-e0c0249ce734","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:16:17.028Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"86e7bd43-813f-436c-b367-b50c7a05828a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:16:17.028Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"86e7bd43-813f-436c-b367-b50c7a05828a","name":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:16:17.028Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"86e7bd43-813f-436c-b367-b50c7a05828a","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:16:17.028Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"86e7bd43-813f-436c-b367-b50c7a05828a"}
{"level":"info","ts":"2026-11-03T15:16:17.028Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"86e7bd43-813f-436c-b367-b50c7a05828a","RayCluster":"counter-8j287","State":""}
{"level":"info","ts":"2026-11-03T15:16:17.028Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"86e7bd43-813f-436c-b367-b50c7a05828a","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:16:20.029Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"f9e00738-4e96-4341-b4ff-9b36c742f813","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:16:20.029Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"f9e00738-4e96-4341-b4ff-9b36c742f813","name":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:16:20.029Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"f9e00738-4e96-4341-b4ff-9b36c742f813","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:16:20.029Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"f9e00738-4e96-4341-b4ff-9b36c742f813"}
{"level":"info","ts":"2026-11-03T15:16:20.029Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"f9e00738-4e96-4341-b4ff-9b36c742f813","RayCluster":"counter-8j287","State":""}
{"level":"info","ts":"2026-11-03T15:16:20.029Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"f9e00738-4e96-4341-b4ff-9b36c742f813","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:16:23.029Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"bf8e145a-34be-45bb-8313-a3a473a4fcf7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:16:23.029Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"bf8e145a-34be-45bb-8313-a3a473a4fcf7","name":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:16:23.029Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"bf8e145a-34be-45bb-8313-a3a473a4fcf7","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:16:23.030Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"bf8e145a-34be-45bb-8313-a3a473a4fcf7"}
{"level":"info","ts":"2026-11-03T15:16:23.030Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"bf8e145a-34be-45bb-8313-a3a473a4fcf7","RayCluster":"counter-8j287","State":""}
{"level":"info","ts":"2026-11-03T15:16:23.030Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"bf8e145a-34be-45bb-8313-a3a473a4fcf7","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:16:24.258Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"79a12884-768e-49fe-8197-1764c6f2b0c7"}
{"level":"info","ts":"2026-11-03T15:16:24.258Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"79a12884-768e-49fe-8197-1764c6f2b0c7"}
{"level":"info","ts":"2026-11-03T15:16:24.258Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"79a12884-768e-49fe-8197-1764c6f2b0c7","1 head service found":"counter-8j287-head-svc"}
{"level":"info","ts":"2026-11-03T15:16:24.258Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"79a12884-768e-49fe-8197-1764c6f2b0c7","Found 1 head Pod":"counter-8j287-head-zldss","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:16:24.258Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"79a12884-768e-49fe-8197-1764c6f2b0c7","head Pod":"counter-8j287-head-zldss","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-8j287-head-zldss. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:16:24.258Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"79a12884-768e-49fe-8197-1764c6f2b0c7","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:16:24.258Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"79a12884-768e-49fe-8197-1764c6f2b0c7","worker Pod":"counter-8j287-small-group-worker-j9tg5","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-8j287-small-group-worker-j9tg5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:16:24.258Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"79a12884-768e-49fe-8197-1764c6f2b0c7","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:16:24.258Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"79a12884-768e-49fe-8197-1764c6f2b0c7","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:16:24.258Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"79a12884-768e-49fe-8197-1764c6f2b0c7","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:16:24.258Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"79a12884-768e-49fe-8197-1764c6f2b0c7","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:16:24.258Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"79a12884-768e-49fe-8197-1764c6f2b0c7","podName":"counter-8j287-head-zldss","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:16:24.258Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"79a12884-768e-49fe-8197-1764c6f2b0c7","podName":"counter-8j287-head-zldss","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:16:24.258Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"79a12884-768e-49fe-8197-1764c6f2b0c7","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:16:24.258Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"79a12884-768e-49fe-8197-1764c6f2b0c7","seconds":300}
{"level":"info","ts":"2026-11-03T15:16:25.269Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"78fb1576-2342-49ff-a18c-ae1acc5c7c58"}
{"level":"info","ts":"2026-11-03T15:16:25.269Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"78fb1576-2342-49ff-a18c-ae1acc5c7c58"}
{"level":"info","ts":"2026-11-03T15:16:25.269Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"78fb1576-2342-49ff-a18c-ae1acc5c7c58","1 head service found":"counter-8j287-head-svc"}
{"level":"info","ts":"2026-11-03T15:16:25.270Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"78fb1576-2342-49ff-a18c-ae1acc5c7c58","Found 1 head Pod":"counter-8j287-head-zldss","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:16:25.270Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"78fb1576-2342-49ff-a18c-ae1acc5c7c58","head Pod":"counter-8j287-head-zldss","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-8j287-head-zldss. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:16:25.270Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"78fb1576-2342-49ff-a18c-ae1acc5c7c58","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:16:25.270Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"78fb1576-2342-49ff-a18c-ae1acc5c7c58","worker Pod":"counter-8j287-small-group-worker-j9tg5","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-8j287-small-group-worker-j9tg5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:16:25.270Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"78fb1576-2342-49ff-a18c-ae1acc5c7c58","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:16:25.270Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"78fb1576-2342-49ff-a18c-ae1acc5c7c58","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:16:25.270Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"78fb1576-2342-49ff-a18c-ae1acc5c7c58","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:16:25.270Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"78fb1576-2342-49ff-a18c-ae1acc5c7c58","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:16:25.270Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"78fb1576-2342-49ff-a18c-ae1acc5c7c58","podName":"counter-8j287-head-zldss","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:14:57Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:16:25.270Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"78fb1576-2342-49ff-a18c-ae1acc5c7c58","podName":"counter-8j287-head-zldss","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:14:57Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:16:25.270Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"78fb1576-2342-49ff-a18c-ae1acc5c7c58","name":"counter-8j287","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:16:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:16:25.331Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"e0a13405-ac78-4c90-bc48-d8305dda5a1d","cluster":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:16:25.332Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"4d737722-bbc4-4c19-9097-72f9da4fa40c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:16:25.332Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"4d737722-bbc4-4c19-9097-72f9da4fa40c","name":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:16:25.332Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"4d737722-bbc4-4c19-9097-72f9da4fa40c","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:16:25.332Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"4d737722-bbc4-4c19-9097-72f9da4fa40c"}
{"level":"info","ts":"2026-11-03T15:16:25.332Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"4d737722-bbc4-4c19-9097-72f9da4fa40c","RayCluster":"counter-8j287","State":""}
{"level":"info","ts":"2026-11-03T15:16:25.332Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"4d737722-bbc4-4c19-9097-72f9da4fa40c","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:14:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:16:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:16:25.332Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"4d737722-bbc4-4c19-9097-72f9da4fa40c","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:16:25.474Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"d4763cde-f6ec-4cd3-bf03-2a86a409f927","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:16:25.474Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"d4763cde-f6ec-4cd3-bf03-2a86a409f927","name":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:16:25.474Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"d4763cde-f6ec-4cd3-bf03-2a86a409f927","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:16:25.474Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"d4763cde-f6ec-4cd3-bf03-2a86a409f927"}
{"level":"info","ts":"2026-11-03T15:16:25.474Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"d4763cde-f6ec-4cd3-bf03-2a86a409f927","RayCluster":"counter-8j287","State":""}
{"level":"info","ts":"2026-11-03T15:16:25.474Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"d4763cde-f6ec-4cd3-bf03-2a86a409f927","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:16:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:16:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:16:27.379Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"bf4dbd51-8a9a-4373-87f3-afed6bf60d10"}
{"level":"info","ts":"2026-11-03T15:16:27.379Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"bf4dbd51-8a9a-4373-87f3-afed6bf60d10"}
{"level":"info","ts":"2026-11-03T15:16:27.379Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"bf4dbd51-8a9a-4373-87f3-afed6bf60d10","1 head service found":"counter-8j287-head-svc"}
{"level":"info","ts":"2026-11-03T15:16:27.379Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"bf4dbd51-8a9a-4373-87f3-afed6bf60d10","Found 1 head Pod":"counter-8j287-head-zldss","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:16:27.379Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"bf4dbd51-8a9a-4373-87f3-afed6bf60d10","head Pod":"counter-8j287-head-zldss","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-8j287-head-zldss. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:16:27.379Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"bf4dbd51-8a9a-4373-87f3-afed6bf60d10","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:16:27.379Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"bf4dbd51-8a9a-4373-87f3-afed6bf60d10","worker Pod":"counter-8j287-small-group-worker-j9tg5","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-8j287-small-group-worker-j9tg5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:16:27.379Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"bf4dbd51-8a9a-4373-87f3-afed6bf60d10","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:16:27.379Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"bf4dbd51-8a9a-4373-87f3-afed6bf60d10","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:16:27.379Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"bf4dbd51-8a9a-4373-87f3-afed6bf60d10","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:16:27.379Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"bf4dbd51-8a9a-4373-87f3-afed6bf60d10","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:16:27.379Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"bf4dbd51-8a9a-4373-87f3-afed6bf60d10","podName":"counter-8j287-head-zldss","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:14:57Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:16:27.379Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"bf4dbd51-8a9a-4373-87f3-afed6bf60d10","podName":"counter-8j287-head-zldss","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:14:57Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:16:27.379Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"bf4dbd51-8a9a-4373-87f3-afed6bf60d10","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:16:27.379Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"bf4dbd51-8a9a-4373-87f3-afed6bf60d10","seconds":300}
{"level":"info","ts":"2026-11-03T15:16:28.475Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"5c83a710-836a-46cb-b7d2-d2cb4e93b9aa","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:16:28.475Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"5c83a710-836a-46cb-b7d2-d2cb4e93b9aa","name":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:16:28.475Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"5c83a710-836a-46cb-b7d2-d2cb4e93b9aa","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:16:28.476Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"5c83a710-836a-46cb-b7d2-d2cb4e93b9aa"}
{"level":"info","ts":"2026-11-03T15:16:28.476Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"5c83a710-836a-46cb-b7d2-d2cb4e93b9aa","RayCluster":"counter-8j287","State":""}
{"level":"info","ts":"2026-11-03T15:16:28.476Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"5c83a710-836a-46cb-b7d2-d2cb4e93b9aa","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:16:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:16:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:16:30.173Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"07b88435-bbf8-401c-af2f-881f3d39957a"}
{"level":"info","ts":"2026-11-03T15:16:30.173Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"07b88435-bbf8-401c-af2f-881f3d39957a"}
{"level":"info","ts":"2026-11-03T15:16:30.173Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"07b88435-bbf8-401c-af2f-881f3d39957a","1 head service found":"counter-8j287-head-svc"}
{"level":"info","ts":"2026-11-03T15:16:30.173Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"07b88435-bbf8-401c-af2f-881f3d39957a","Found 1 head Pod":"counter-8j287-head-zldss","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:16:30.173Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"07b88435-bbf8-401c-af2f-881f3d39957a","head Pod":"counter-8j287-head-zldss","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-8j287-head-zldss. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:16:30.173Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"07b88435-bbf8-401c-af2f-881f3d39957a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:16:30.173Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"07b88435-bbf8-401c-af2f-881f3d39957a","worker Pod":"counter-8j287-small-group-worker-j9tg5","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-8j287-small-group-worker-j9tg5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:16:30.173Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"07b88435-bbf8-401c-af2f-881f3d39957a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:16:30.173Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"07b88435-bbf8-401c-af2f-881f3d39957a","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:16:30.173Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"07b88435-bbf8-401c-af2f-881f3d39957a","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:16:30.173Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"07b88435-bbf8-401c-af2f-881f3d39957a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:16:30.173Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"07b88435-bbf8-401c-af2f-881f3d39957a","podName":"counter-8j287-head-zldss","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:14:57Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:16:30.173Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"07b88435-bbf8-401c-af2f-881f3d39957a","podName":"counter-8j287-head-zldss","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:14:57Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:16:30.173Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"07b88435-bbf8-401c-af2f-881f3d39957a","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:16:30.173Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"07b88435-bbf8-401c-af2f-881f3d39957a","seconds":300}
{"level":"info","ts":"2026-11-03T15:16:31.179Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"02255722-1245-4e66-b9ea-2dd9ac50c34f"}
{"level":"info","ts":"2026-11-03T15:16:31.179Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"02255722-1245-4e66-b9ea-2dd9ac50c34f"}
{"level":"info","ts":"2026-11-03T15:16:31.179Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"02255722-1245-4e66-b9ea-2dd9ac50c34f","1 head service found":"counter-8j287-head-svc"}
{"level":"info","ts":"2026-11-03T15:16:31.179Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"02255722-1245-4e66-b9ea-2dd9ac50c34f","Found 1 head Pod":"counter-8j287-head-zldss","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:16:31.179Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"02255722-1245-4e66-b9ea-2dd9ac50c34f","head Pod":"counter-8j287-head-zldss","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-8j287-head-zldss. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:16:31.179Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"02255722-1245-4e66-b9ea-2dd9ac50c34f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:16:31.179Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"02255722-1245-4e66-b9ea-2dd9ac50c34f","worker Pod":"counter-8j287-small-group-worker-j9tg5","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-8j287-small-group-worker-j9tg5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:16:31.179Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"02255722-1245-4e66-b9ea-2dd9ac50c34f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:16:31.179Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"02255722-1245-4e66-b9ea-2dd9ac50c34f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:16:31.179Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"02255722-1245-4e66-b9ea-2dd9ac50c34f","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:16:31.179Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"02255722-1245-4e66-b9ea-2dd9ac50c34f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:16:31.179Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"02255722-1245-4e66-b9ea-2dd9ac50c34f","podName":"counter-8j287-head-zldss","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:14:57Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:16:31.179Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"02255722-1245-4e66-b9ea-2dd9ac50c34f","podName":"counter-8j287-head-zldss","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:14:57Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:16:31.179Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"02255722-1245-4e66-b9ea-2dd9ac50c34f","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:16:31.179Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"02255722-1245-4e66-b9ea-2dd9ac50c34f","seconds":300}
{"level":"info","ts":"2026-11-03T15:16:31.476Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"08155211-a9b8-4a6c-b911-fe319dd3ea5b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:16:31.476Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"08155211-a9b8-4a6c-b911-fe319dd3ea5b","name":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:16:31.476Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"08155211-a9b8-4a6c-b911-fe319dd3ea5b","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:16:31.477Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"08155211-a9b8-4a6c-b911-fe319dd3ea5b"}
{"level":"info","ts":"2026-11-03T15:16:31.477Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"08155211-a9b8-4a6c-b911-fe319dd3ea5b","RayCluster":"counter-8j287","State":""}
{"level":"info","ts":"2026-11-03T15:16:31.477Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"08155211-a9b8-4a6c-b911-fe319dd3ea5b","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:16:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:16:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:16:34.478Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"f2390932-1026-4fce-ac46-8f178d2f7482","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:16:34.478Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"f2390932-1026-4fce-ac46-8f178d2f7482","name":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:16:34.478Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"f2390932-1026-4fce-ac46-8f178d2f7482","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:16:34.478Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"f2390932-1026-4fce-ac46-8f178d2f7482"}
{"level":"info","ts":"2026-11-03T15:16:34.478Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"f2390932-1026-4fce-ac46-8f178d2f7482","RayCluster":"counter-8j287","State":""}
{"level":"info","ts":"2026-11-03T15:16:34.478Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"f2390932-1026-4fce-ac46-8f178d2f7482","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:16:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:16:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:16:36.380Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"526cbdda-f6ee-41aa-997a-66084b9dd7d3"}
{"level":"info","ts":"2026-11-03T15:16:36.380Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"526cbdda-f6ee-41aa-997a-66084b9dd7d3"}
{"level":"info","ts":"2026-11-03T15:16:36.380Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"526cbdda-f6ee-41aa-997a-66084b9dd7d3","1 head service found":"counter-8j287-head-svc"}
{"level":"info","ts":"2026-11-03T15:16:36.380Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"526cbdda-f6ee-41aa-997a-66084b9dd7d3","Found 1 head Pod":"counter-8j287-head-zldss","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:16:36.380Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"526cbdda-f6ee-41aa-997a-66084b9dd7d3","head Pod":"counter-8j287-head-zldss","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-8j287-head-zldss. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:16:36.380Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"526cbdda-f6ee-41aa-997a-66084b9dd7d3","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:16:36.380Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"526cbdda-f6ee-41aa-997a-66084b9dd7d3","worker Pod":"counter-8j287-small-group-worker-j9tg5","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-8j287-small-group-worker-j9tg5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:16:36.380Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"526cbdda-f6ee-41aa-997a-66084b9dd7d3","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:16:36.380Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"526cbdda-f6ee-41aa-997a-66084b9dd7d3","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:16:36.380Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"526cbdda-f6ee-41aa-997a-66084b9dd7d3","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:16:36.380Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"526cbdda-f6ee-41aa-997a-66084b9dd7d3","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:16:36.380Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"526cbdda-f6ee-41aa-997a-66084b9dd7d3","podName":"counter-8j287-small-group-worker-j9tg5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:16:36.380Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"526cbdda-f6ee-41aa-997a-66084b9dd7d3","podName":"counter-8j287-small-group-worker-j9tg5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:16:36.380Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"526cbdda-f6ee-41aa-997a-66084b9dd7d3","name":"counter-8j287","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:16:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:16:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:16:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:16:36.480Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"87dfecf7-4fda-4e7e-bb2c-300c9a8f74dd","cluster":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:16:36.480Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"e1601395-00ae-4d9d-8964-9e4c75172e8b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:16:36.480Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"e1601395-00ae-4d9d-8964-9e4c75172e8b","name":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:16:36.480Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"e1601395-00ae-4d9d-8964-9e4c75172e8b","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:16:36.480Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"e1601395-00ae-4d9d-8964-9e4c75172e8b"}
{"level":"info","ts":"2026-11-03T15:16:36.480Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"e1601395-00ae-4d9d-8964-9e4c75172e8b","RayCluster":"counter-8j287","State":""}
{"level":"info","ts":"2026-11-03T15:16:36.480Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"e1601395-00ae-4d9d-8964-9e4c75172e8b","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:16:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:16:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:16:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:16:36.480Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"e1601395-00ae-4d9d-8964-9e4c75172e8b","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:16:36.581Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"4d3b82de-7a0e-434d-8ade-b78891944794","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:16:36.581Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"4d3b82de-7a0e-434d-8ade-b78891944794","name":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:16:36.581Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"4d3b82de-7a0e-434d-8ade-b78891944794","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:16:36.581Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"4d3b82de-7a0e-434d-8ade-b78891944794"}
{"level":"info","ts":"2026-11-03T15:16:36.581Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"4d3b82de-7a0e-434d-8ade-b78891944794","RayCluster":"counter-8j287","State":""}
{"level":"info","ts":"2026-11-03T15:16:36.581Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"4d3b82de-7a0e-434d-8ade-b78891944794","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:16:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:16:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:16:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:16:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:16:38.480Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"c24b4417-469e-4509-a387-88f893f18d16"}
{"level":"info","ts":"2026-11-03T15:16:38.480Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"c24b4417-469e-4509-a387-88f893f18d16"}
{"level":"info","ts":"2026-11-03T15:16:38.480Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"c24b4417-469e-4509-a387-88f893f18d16","1 head service found":"counter-8j287-head-svc"}
{"level":"info","ts":"2026-11-03T15:16:38.481Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"c24b4417-469e-4509-a387-88f893f18d16","Found 1 head Pod":"counter-8j287-head-zldss","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:16:38.481Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"c24b4417-469e-4509-a387-88f893f18d16","head Pod":"counter-8j287-head-zldss","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-8j287-head-zldss. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:16:38.481Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"c24b4417-469e-4509-a387-88f893f18d16","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:16:38.481Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"c24b4417-469e-4509-a387-88f893f18d16","worker Pod":"counter-8j287-small-group-worker-j9tg5","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-8j287-small-group-worker-j9tg5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:16:38.481Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"c24b4417-469e-4509-a387-88f893f18d16","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:16:38.481Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"c24b4417-469e-4509-a387-88f893f18d16","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:16:38.481Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"c24b4417-469e-4509-a387-88f893f18d16","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:16:38.481Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"c24b4417-469e-4509-a387-88f893f18d16","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:16:38.481Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"c24b4417-469e-4509-a387-88f893f18d16","podName":"counter-8j287-small-group-worker-j9tg5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:16:38.481Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"c24b4417-469e-4509-a387-88f893f18d16","podName":"counter-8j287-small-group-worker-j9tg5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:16:38.481Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"c24b4417-469e-4509-a387-88f893f18d16","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:16:38.481Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"c24b4417-469e-4509-a387-88f893f18d16","seconds":300}
{"level":"info","ts":"2026-11-03T15:16:39.582Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"29e85f1d-f68e-43f4-9d94-f25a25b9e243","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:16:39.582Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"29e85f1d-f68e-43f4-9d94-f25a25b9e243","name":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:16:39.582Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"29e85f1d-f68e-43f4-9d94-f25a25b9e243","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:16:39.582Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"29e85f1d-f68e-43f4-9d94-f25a25b9e243"}
{"level":"info","ts":"2026-11-03T15:16:39.582Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"29e85f1d-f68e-43f4-9d94-f25a25b9e243","RayCluster":"counter-8j287","State":""}
{"level":"info","ts":"2026-11-03T15:16:39.582Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"29e85f1d-f68e-43f4-9d94-f25a25b9e243","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:16:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:16:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:16:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:16:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:16:40.227Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"4e718aa0-4479-4bf4-b170-7577ebd415bd"}
{"level":"info","ts":"2026-11-03T15:16:40.227Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"4e718aa0-4479-4bf4-b170-7577ebd415bd"}
{"level":"info","ts":"2026-11-03T15:16:40.227Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"4e718aa0-4479-4bf4-b170-7577ebd415bd","1 head service found":"counter-8j287-head-svc"}
{"level":"info","ts":"2026-11-03T15:16:40.227Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"4e718aa0-4479-4bf4-b170-7577ebd415bd","Found 1 head Pod":"counter-8j287-head-zldss","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:16:40.227Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"4e718aa0-4479-4bf4-b170-7577ebd415bd","head Pod":"counter-8j287-head-zldss","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-8j287-head-zldss. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:16:40.227Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"4e718aa0-4479-4bf4-b170-7577ebd415bd","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:16:40.227Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"4e718aa0-4479-4bf4-b170-7577ebd415bd","worker Pod":"counter-8j287-small-group-worker-j9tg5","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-8j287-small-group-worker-j9tg5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:16:40.228Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"4e718aa0-4479-4bf4-b170-7577ebd415bd","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:16:40.228Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"4e718aa0-4479-4bf4-b170-7577ebd415bd","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:16:40.228Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"4e718aa0-4479-4bf4-b170-7577ebd415bd","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:16:40.228Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"4e718aa0-4479-4bf4-b170-7577ebd415bd","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:16:40.228Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"4e718aa0-4479-4bf4-b170-7577ebd415bd","podName":"counter-8j287-small-group-worker-j9tg5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:16:40.228Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"4e718aa0-4479-4bf4-b170-7577ebd415bd","podName":"counter-8j287-small-group-worker-j9tg5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:16:40.228Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"4e718aa0-4479-4bf4-b170-7577ebd415bd","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:16:40.228Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"4e718aa0-4479-4bf4-b170-7577ebd415bd","seconds":300}
{"level":"info","ts":"2026-11-03T15:16:41.221Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"dcef9e0c-4812-4550-88e1-b4ee98bf7c91"}
{"level":"info","ts":"2026-11-03T15:16:41.221Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"dcef9e0c-4812-4550-88e1-b4ee98bf7c91"}
{"level":"info","ts":"2026-11-03T15:16:41.221Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"dcef9e0c-4812-4550-88e1-b4ee98bf7c91","1 head service found":"counter-8j287-head-svc"}
{"level":"info","ts":"2026-11-03T15:16:41.221Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"dcef9e0c-4812-4550-88e1-b4ee98bf7c91","Found 1 head Pod":"counter-8j287-head-zldss","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:16:41.221Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"dcef9e0c-4812-4550-88e1-b4ee98bf7c91","head Pod":"counter-8j287-head-zldss","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-8j287-head-zldss. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:16:41.221Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"dcef9e0c-4812-4550-88e1-b4ee98bf7c91","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:16:41.221Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"dcef9e0c-4812-4550-88e1-b4ee98bf7c91","worker Pod":"counter-8j287-small-group-worker-j9tg5","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-8j287-small-group-worker-j9tg5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:16:41.221Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"dcef9e0c-4812-4550-88e1-b4ee98bf7c91","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:16:41.221Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"dcef9e0c-4812-4550-88e1-b4ee98bf7c91","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:16:41.221Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"dcef9e0c-4812-4550-88e1-b4ee98bf7c91","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:16:41.221Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"dcef9e0c-4812-4550-88e1-b4ee98bf7c91","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:16:41.221Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"dcef9e0c-4812-4550-88e1-b4ee98bf7c91","podName":"counter-8j287-small-group-worker-j9tg5","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:14:57Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:16:41.221Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"dcef9e0c-4812-4550-88e1-b4ee98bf7c91","podName":"counter-8j287-small-group-worker-j9tg5","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:14:57Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:16:41.221Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"dcef9e0c-4812-4550-88e1-b4ee98bf7c91","name":"counter-8j287","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:16:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:16:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:16:41Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:16:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:16:41.283Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"3708695d-a93c-467f-8c16-76813542c8b9","cluster":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:16:41.283Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"793339e1-3d62-45be-a0d0-e43f9d1e9688","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:16:41.283Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"793339e1-3d62-45be-a0d0-e43f9d1e9688","name":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:16:41.283Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"793339e1-3d62-45be-a0d0-e43f9d1e9688","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:16:41.283Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"793339e1-3d62-45be-a0d0-e43f9d1e9688"}
{"level":"info","ts":"2026-11-03T15:16:41.283Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"793339e1-3d62-45be-a0d0-e43f9d1e9688","RayCluster":"counter-8j287","State":""}
{"level":"info","ts":"2026-11-03T15:16:41.283Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"793339e1-3d62-45be-a0d0-e43f9d1e9688","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:16:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:16:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:16:41Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:16:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:16:41.283Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"793339e1-3d62-45be-a0d0-e43f9d1e9688","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:16:41.380Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"06b5d3c9-7ec0-45e6-9172-81257accb762","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:16:41.380Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"06b5d3c9-7ec0-45e6-9172-81257accb762","name":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:16:41.380Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"06b5d3c9-7ec0-45e6-9172-81257accb762","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:16:41.380Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"06b5d3c9-7ec0-45e6-9172-81257accb762"}
{"level":"info","ts":"2026-11-03T15:16:41.380Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"06b5d3c9-7ec0-45e6-9172-81257accb762","RayCluster":"counter-8j287","State":""}
{"level":"info","ts":"2026-11-03T15:16:41.380Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"06b5d3c9-7ec0-45e6-9172-81257accb762","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:16:41Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:16:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:16:41Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:16:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"error","ts":"2026-11-03T15:16:42.266Z","logger":"events","msg":"Server rejected event (will not retry!)","event":{"namespace":"test-ns-mvz6h","name":"counter-auth-kk8bt.18e4918495491fad"},"error":"namespaces \"test-ns-mvz6h\" not found","stacktrace":"k8s.io/client-go/tools/events.recordEvent\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:270\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).finishSeries\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:153\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext.func1\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:255\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:256\nk8s.io/apimachinery/pkg/util/wait.JitterUntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:223\nk8s.io/apimachinery/pkg/util/wait.UntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:172"}
{"level":"error","ts":"2026-11-03T15:16:42.417Z","logger":"events","msg":"Server rejected event (will not retry!)","event":{"namespace":"test-ns-mvz6h","name":"stop-254kg.18e49165e8aad938"},"error":"namespaces \"test-ns-mvz6h\" not found","stacktrace":"k8s.io/client-go/tools/events.recordEvent\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:270\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).finishSeries\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:153\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext.func1\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:255\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:256\nk8s.io/apimachinery/pkg/util/wait.JitterUntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:223\nk8s.io/apimachinery/pkg/util/wait.UntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:172"}
{"level":"error","ts":"2026-11-03T15:16:42.569Z","logger":"events","msg":"Server rejected event (will not retry!)","event":{"namespace":"test-ns-mvz6h","name":"fail-g56b8.18e4914a218f28c1"},"error":"namespaces \"test-ns-mvz6h\" not found","stacktrace":"k8s.io/client-go/tools/events.recordEvent\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:270\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).finishSeries\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:153\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext.func1\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:255\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:256\nk8s.io/apimachinery/pkg/util/wait.JitterUntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:223\nk8s.io/apimachinery/pkg/util/wait.UntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:172"}
{"level":"error","ts":"2026-11-03T15:16:42.721Z","logger":"events","msg":"Server rejected event (will not retry!)","event":{"namespace":"test-ns-mvz6h","name":"fail-g56b8.18e4914b6f88c159"},"error":"namespaces \"test-ns-mvz6h\" not found","stacktrace":"k8s.io/client-go/tools/events.recordEvent\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:270\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).finishSeries\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:153\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext.func1\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:255\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:256\nk8s.io/apimachinery/pkg/util/wait.JitterUntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:223\nk8s.io/apimachinery/pkg/util/wait.UntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:172"}
{"level":"error","ts":"2026-11-03T15:16:42.872Z","logger":"events","msg":"Server rejected event (will not retry!)","event":{"namespace":"test-ns-mvz6h","name":"counter-auth-kk8bt.18e49185e25e5715"},"error":"namespaces \"test-ns-mvz6h\" not found","stacktrace":"k8s.io/client-go/tools/events.recordEvent\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:270\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).finishSeries\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:153\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext.func1\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:255\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:256\nk8s.io/apimachinery/pkg/util/wait.JitterUntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:223\nk8s.io/apimachinery/pkg/util/wait.UntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:172"}
{"level":"error","ts":"2026-11-03T15:16:43.023Z","logger":"events","msg":"Server rejected event (will not retry!)","event":{"namespace":"test-ns-mvz6h","name":"stop-254kg.18e49167cf99b8a5"},"error":"namespaces \"test-ns-mvz6h\" not found","stacktrace":"k8s.io/client-go/tools/events.recordEvent\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:270\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).finishSeries\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:153\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext.func1\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:255\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:256\nk8s.io/apimachinery/pkg/util/wait.JitterUntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:223\nk8s.io/apimachinery/pkg/util/wait.UntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:172"}
{"level":"error","ts":"2026-11-03T15:16:43.175Z","logger":"events","msg":"Server rejected event (will not retry!)","event":{"namespace":"test-ns-mvz6h","name":"stop-254kg.18e49167b732e913"},"error":"namespaces \"test-ns-mvz6h\" not found","stacktrace":"k8s.io/client-go/tools/events.recordEvent\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:270\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).finishSeries\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:153\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext.func1\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:255\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:256\nk8s.io/apimachinery/pkg/util/wait.JitterUntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:223\nk8s.io/apimachinery/pkg/util/wait.UntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:172"}
{"level":"info","ts":"2026-11-03T15:16:43.318Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"22cce93f-e020-4ac2-9d0c-9b0f6c817a19"}
{"level":"info","ts":"2026-11-03T15:16:43.318Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"22cce93f-e020-4ac2-9d0c-9b0f6c817a19"}
{"level":"info","ts":"2026-11-03T15:16:43.318Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"22cce93f-e020-4ac2-9d0c-9b0f6c817a19","1 head service found":"counter-8j287-head-svc"}
{"level":"info","ts":"2026-11-03T15:16:43.318Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"22cce93f-e020-4ac2-9d0c-9b0f6c817a19","Found 1 head Pod":"counter-8j287-head-zldss","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:16:43.318Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"22cce93f-e020-4ac2-9d0c-9b0f6c817a19","head Pod":"counter-8j287-head-zldss","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-8j287-head-zldss. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:16:43.318Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"22cce93f-e020-4ac2-9d0c-9b0f6c817a19","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:16:43.318Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"22cce93f-e020-4ac2-9d0c-9b0f6c817a19","worker Pod":"counter-8j287-small-group-worker-j9tg5","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-8j287-small-group-worker-j9tg5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:16:43.318Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"22cce93f-e020-4ac2-9d0c-9b0f6c817a19","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:16:43.318Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"22cce93f-e020-4ac2-9d0c-9b0f6c817a19","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:16:43.318Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"22cce93f-e020-4ac2-9d0c-9b0f6c817a19","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:16:43.318Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"22cce93f-e020-4ac2-9d0c-9b0f6c817a19","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:16:43.318Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"22cce93f-e020-4ac2-9d0c-9b0f6c817a19","podName":"counter-8j287-small-group-worker-j9tg5","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:14:57Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:16:43.318Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"22cce93f-e020-4ac2-9d0c-9b0f6c817a19","podName":"counter-8j287-small-group-worker-j9tg5","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:14:57Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:16:43.318Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"22cce93f-e020-4ac2-9d0c-9b0f6c817a19","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:16:43.318Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"22cce93f-e020-4ac2-9d0c-9b0f6c817a19","seconds":300}
{"level":"info","ts":"2026-11-03T15:16:44.381Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"03d1cdf6-7922-4f69-8050-46d7aa931992","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:16:44.381Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"03d1cdf6-7922-4f69-8050-46d7aa931992","name":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:16:44.381Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"03d1cdf6-7922-4f69-8050-46d7aa931992","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:16:44.381Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"03d1cdf6-7922-4f69-8050-46d7aa931992"}
{"level":"info","ts":"2026-11-03T15:16:44.381Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"03d1cdf6-7922-4f69-8050-46d7aa931992","RayCluster":"counter-8j287","State":""}
{"level":"info","ts":"2026-11-03T15:16:44.381Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"03d1cdf6-7922-4f69-8050-46d7aa931992","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:16:41Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:16:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:16:41Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:16:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:16:47.382Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"0f6a522b-62ea-4610-b8cd-c318708c2e78","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:16:47.382Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"0f6a522b-62ea-4610-b8cd-c318708c2e78","name":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:16:47.382Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"0f6a522b-62ea-4610-b8cd-c318708c2e78","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:16:47.383Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"0f6a522b-62ea-4610-b8cd-c318708c2e78"}
{"level":"info","ts":"2026-11-03T15:16:47.383Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"0f6a522b-62ea-4610-b8cd-c318708c2e78","RayCluster":"counter-8j287","State":""}
{"level":"info","ts":"2026-11-03T15:16:47.383Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"0f6a522b-62ea-4610-b8cd-c318708c2e78","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:16:41Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:16:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:16:41Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:16:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:16:50.383Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"5f703354-517c-4bf8-94c2-f514894b6968","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:16:50.383Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"5f703354-517c-4bf8-94c2-f514894b6968","name":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:16:50.384Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"5f703354-517c-4bf8-94c2-f514894b6968","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:16:50.384Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"5f703354-517c-4bf8-94c2-f514894b6968"}
{"level":"info","ts":"2026-11-03T15:16:50.384Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"5f703354-517c-4bf8-94c2-f514894b6968","RayCluster":"counter-8j287","State":""}
{"level":"info","ts":"2026-11-03T15:16:50.384Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"5f703354-517c-4bf8-94c2-f514894b6968","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:16:41Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:16:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:16:41Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:16:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:16:52.229Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"88934792-19ca-44f3-94a4-6237f96f4f99"}
{"level":"info","ts":"2026-11-03T15:16:52.229Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"88934792-19ca-44f3-94a4-6237f96f4f99"}
{"level":"info","ts":"2026-11-03T15:16:52.229Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"88934792-19ca-44f3-94a4-6237f96f4f99","1 head service found":"counter-8j287-head-svc"}
{"level":"info","ts":"2026-11-03T15:16:52.229Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"88934792-19ca-44f3-94a4-6237f96f4f99","Found 1 head Pod":"counter-8j287-head-zldss","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:16:52.229Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"88934792-19ca-44f3-94a4-6237f96f4f99","head Pod":"counter-8j287-head-zldss","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-8j287-head-zldss. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:16:52.229Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"88934792-19ca-44f3-94a4-6237f96f4f99","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:16:52.229Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"88934792-19ca-44f3-94a4-6237f96f4f99","worker Pod":"counter-8j287-small-group-worker-j9tg5","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-8j287-small-group-worker-j9tg5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:16:52.229Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"88934792-19ca-44f3-94a4-6237f96f4f99","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:16:52.229Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"88934792-19ca-44f3-94a4-6237f96f4f99","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:16:52.229Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"88934792-19ca-44f3-94a4-6237f96f4f99","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:16:52.229Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"88934792-19ca-44f3-94a4-6237f96f4f99","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:16:52.229Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"88934792-19ca-44f3-94a4-6237f96f4f99","name":"counter-8j287","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:16:41Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:16:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:16:52Z","stateTransitionTimes":{"ready":"2026-11-03T15:16:52Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:16:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:16:52Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:16:52.295Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"556495ee-9192-444c-9d7c-52436dacb8f0","cluster":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:16:52.295Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"ed1313ee-72bb-4422-beef-970a869b9dc4","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:16:52.295Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"ed1313ee-72bb-4422-beef-970a869b9dc4","name":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:16:52.295Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"ed1313ee-72bb-4422-beef-970a869b9dc4","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:16:52.295Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"ed1313ee-72bb-4422-beef-970a869b9dc4"}
{"level":"info","ts":"2026-11-03T15:16:52.357Z","logger":"controllers.RayJob","msg":"Created submitter Kubernetes Job for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"ed1313ee-72bb-4422-beef-970a869b9dc4","Kubernetes Job":"counter"}
{"level":"info","ts":"2026-11-03T15:16:52.357Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"ed1313ee-72bb-4422-beef-970a869b9dc4","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:16:41Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:16:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","dashboardURL":"counter-8j287-head-svc.test-ns-5lqjm.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:16:41Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:16:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:16:52.358Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"ed1313ee-72bb-4422-beef-970a869b9dc4","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-11-03T15:16:52.443Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"b1e0dc26-5ced-4487-b5eb-6b35f5087d79","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:16:52.443Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"b1e0dc26-5ced-4487-b5eb-6b35f5087d79","name":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:16:52.443Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"b1e0dc26-5ced-4487-b5eb-6b35f5087d79","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:16:52.443Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"b1e0dc26-5ced-4487-b5eb-6b35f5087d79"}
{"level":"info","ts":"2026-11-03T15:16:52.443Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job for RayJob already exists","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"b1e0dc26-5ced-4487-b5eb-6b35f5087d79","Kubernetes Job":"counter"}
{"level":"info","ts":"2026-11-03T15:16:52.443Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"b1e0dc26-5ced-4487-b5eb-6b35f5087d79","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:16:41Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:16:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","dashboardURL":"counter-8j287-head-svc.test-ns-5lqjm.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:16:41Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:16:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:16:52.443Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"b1e0dc26-5ced-4487-b5eb-6b35f5087d79","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-11-03T15:16:52.497Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"b1e0dc26-5ced-4487-b5eb-6b35f5087d79","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-11-03T15:16:52.497Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"b1e0dc26-5ced-4487-b5eb-6b35f5087d79"}
{"level":"error","ts":"2026-11-03T15:16:52.497Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"b1e0dc26-5ced-4487-b5eb-6b35f5087d79","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:16:52.497Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"b3bacc3b-9cee-4afc-a4ba-1bcbf9dfa61c","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:16:52.497Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"b3bacc3b-9cee-4afc-a4ba-1bcbf9dfa61c","name":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:16:52.497Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"b3bacc3b-9cee-4afc-a4ba-1bcbf9dfa61c","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:16:52.498Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"b3bacc3b-9cee-4afc-a4ba-1bcbf9dfa61c"}
{"level":"info","ts":"2026-11-03T15:16:52.498Z","logger":"controllers.RayJob","msg":"Create a Kubernetes Service","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"b3bacc3b-9cee-4afc-a4ba-1bcbf9dfa61c","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:16:52.560Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"53e5a44c-53b1-4875-b572-59fde04ab569","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:16:52.560Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"53e5a44c-53b1-4875-b572-59fde04ab569","name":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:16:52.560Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"53e5a44c-53b1-4875-b572-59fde04ab569","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:16:52.560Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"53e5a44c-53b1-4875-b572-59fde04ab569"}
{"level":"info","ts":"2026-11-03T15:16:52.560Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"53e5a44c-53b1-4875-b572-59fde04ab569","rayCluster":"counter-8j287","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:16:54.295Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"999f5d36-f67f-40a0-bdfa-db4f6ed18ffb"}
{"level":"info","ts":"2026-11-03T15:16:54.295Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"999f5d36-f67f-40a0-bdfa-db4f6ed18ffb"}
{"level":"info","ts":"2026-11-03T15:16:54.296Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"999f5d36-f67f-40a0-bdfa-db4f6ed18ffb","1 head service found":"counter-8j287-head-svc"}
{"level":"info","ts":"2026-11-03T15:16:54.296Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"999f5d36-f67f-40a0-bdfa-db4f6ed18ffb","Found 1 head Pod":"counter-8j287-head-zldss","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:16:54.296Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"999f5d36-f67f-40a0-bdfa-db4f6ed18ffb","head Pod":"counter-8j287-head-zldss","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-8j287-head-zldss. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:16:54.296Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"999f5d36-f67f-40a0-bdfa-db4f6ed18ffb","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:16:54.296Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"999f5d36-f67f-40a0-bdfa-db4f6ed18ffb","worker Pod":"counter-8j287-small-group-worker-j9tg5","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-8j287-small-group-worker-j9tg5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:16:54.296Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"999f5d36-f67f-40a0-bdfa-db4f6ed18ffb","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:16:54.296Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"999f5d36-f67f-40a0-bdfa-db4f6ed18ffb","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:16:54.296Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"999f5d36-f67f-40a0-bdfa-db4f6ed18ffb","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:16:54.296Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"999f5d36-f67f-40a0-bdfa-db4f6ed18ffb","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:16:54.296Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"999f5d36-f67f-40a0-bdfa-db4f6ed18ffb","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:16:54.296Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"999f5d36-f67f-40a0-bdfa-db4f6ed18ffb","seconds":300}
{"level":"info","ts":"2026-11-03T15:16:55.230Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"41e03be3-761d-4fb3-8bfe-cf7c3fca0e9b","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:16:55.230Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"41e03be3-761d-4fb3-8bfe-cf7c3fca0e9b","name":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:16:55.230Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"41e03be3-761d-4fb3-8bfe-cf7c3fca0e9b","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:16:55.230Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"41e03be3-761d-4fb3-8bfe-cf7c3fca0e9b"}
{"level":"info","ts":"2026-11-03T15:16:55.230Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"41e03be3-761d-4fb3-8bfe-cf7c3fca0e9b","rayCluster":"counter-8j287","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:16:58.236Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"9078a617-f711-4781-a2e0-78475550bac1","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:16:58.236Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"9078a617-f711-4781-a2e0-78475550bac1","name":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:16:58.236Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"9078a617-f711-4781-a2e0-78475550bac1","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:16:58.236Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"9078a617-f711-4781-a2e0-78475550bac1"}
{"level":"info","ts":"2026-11-03T15:16:58.236Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"9078a617-f711-4781-a2e0-78475550bac1","rayCluster":"counter-8j287","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:17:01.243Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"fbe53e37-664d-41d4-9e9f-755217d4823e","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:17:01.243Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"fbe53e37-664d-41d4-9e9f-755217d4823e","name":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:17:01.243Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"fbe53e37-664d-41d4-9e9f-755217d4823e","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:17:01.243Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"fbe53e37-664d-41d4-9e9f-755217d4823e"}
{"level":"info","ts":"2026-11-03T15:17:01.243Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"fbe53e37-664d-41d4-9e9f-755217d4823e","rayCluster":"counter-8j287","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:17:01.251Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"fbe53e37-664d-41d4-9e9f-755217d4823e","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","dashboardURL":"counter-8j287-head-svc.test-ns-5lqjm.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:16:41Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:16:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:17:00Z"},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","dashboardURL":"counter-8j287-head-svc.test-ns-5lqjm.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:16:52Z","stateTransitionTimes":{"ready":"2026-11-03T15:16:52Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:16:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:16:52Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:17:01.251Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"fbe53e37-664d-41d4-9e9f-755217d4823e","old JobStatus":"","new JobStatus":"PENDING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-11-03T15:17:01.355Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"d9e337e4-a541-4f95-8c91-44cf1f758c2a","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:17:01.355Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"d9e337e4-a541-4f95-8c91-44cf1f758c2a","name":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:17:01.355Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"d9e337e4-a541-4f95-8c91-44cf1f758c2a","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:17:01.355Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"d9e337e4-a541-4f95-8c91-44cf1f758c2a"}
{"level":"info","ts":"2026-11-03T15:17:01.355Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"d9e337e4-a541-4f95-8c91-44cf1f758c2a","rayCluster":"counter-8j287","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:17:01.358Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"d9e337e4-a541-4f95-8c91-44cf1f758c2a","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:17:00Z"},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","dashboardURL":"counter-8j287-head-svc.test-ns-5lqjm.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:16:52Z","stateTransitionTimes":{"ready":"2026-11-03T15:16:52Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:16:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:16:52Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:17:00Z"},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","dashboardURL":"counter-8j287-head-svc.test-ns-5lqjm.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:16:52Z","stateTransitionTimes":{"ready":"2026-11-03T15:16:52Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:16:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:16:52Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:17:04.359Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"5c13b6ad-c0b5-4ebe-be14-4f5945c48585","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:17:04.359Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"5c13b6ad-c0b5-4ebe-be14-4f5945c48585","name":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:17:04.359Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"5c13b6ad-c0b5-4ebe-be14-4f5945c48585","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:17:04.359Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"5c13b6ad-c0b5-4ebe-be14-4f5945c48585"}
{"level":"info","ts":"2026-11-03T15:17:04.359Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"5c13b6ad-c0b5-4ebe-be14-4f5945c48585","rayCluster":"counter-8j287","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:17:04.362Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"5c13b6ad-c0b5-4ebe-be14-4f5945c48585","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:17:00Z"},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","dashboardURL":"counter-8j287-head-svc.test-ns-5lqjm.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:16:52Z","stateTransitionTimes":{"ready":"2026-11-03T15:16:52Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:16:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:16:52Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:17:00Z"},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","dashboardURL":"counter-8j287-head-svc.test-ns-5lqjm.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:16:52Z","stateTransitionTimes":{"ready":"2026-11-03T15:16:52Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:16:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:16:52Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:17:04.363Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"5c13b6ad-c0b5-4ebe-be14-4f5945c48585","old JobStatus":"PENDING","new JobStatus":"RUNNING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-11-03T15:17:04.460Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"f04d14fc-2a39-42a0-9a46-e9593900c6e2","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:17:04.460Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"f04d14fc-2a39-42a0-9a46-e9593900c6e2","name":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:17:04.460Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"f04d14fc-2a39-42a0-9a46-e9593900c6e2","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:17:04.460Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"f04d14fc-2a39-42a0-9a46-e9593900c6e2"}
{"level":"info","ts":"2026-11-03T15:17:04.460Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"f04d14fc-2a39-42a0-9a46-e9593900c6e2","rayCluster":"counter-8j287","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:17:04.463Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"f04d14fc-2a39-42a0-9a46-e9593900c6e2","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:17:00Z"},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","dashboardURL":"counter-8j287-head-svc.test-ns-5lqjm.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:16:52Z","stateTransitionTimes":{"ready":"2026-11-03T15:16:52Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:16:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:16:52Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:17:00Z"},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","dashboardURL":"counter-8j287-head-svc.test-ns-5lqjm.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:16:52Z","stateTransitionTimes":{"ready":"2026-11-03T15:16:52Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:16:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:16:52Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:17:07.464Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"c7a30953-a962-4d09-b791-b6904b02a4f4","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:17:07.464Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"c7a30953-a962-4d09-b791-b6904b02a4f4","name":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:17:07.464Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"c7a30953-a962-4d09-b791-b6904b02a4f4","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:17:07.464Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"c7a30953-a962-4d09-b791-b6904b02a4f4"}
{"level":"info","ts":"2026-11-03T15:17:07.464Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"c7a30953-a962-4d09-b791-b6904b02a4f4","rayCluster":"counter-8j287","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:17:07.468Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"c7a30953-a962-4d09-b791-b6904b02a4f4","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:17:00Z"},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","dashboardURL":"counter-8j287-head-svc.test-ns-5lqjm.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:16:52Z","stateTransitionTimes":{"ready":"2026-11-03T15:16:52Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:16:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:16:52Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:17:00Z","endTime":"2026-11-03T15:17:04Z"},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","dashboardURL":"counter-8j287-head-svc.test-ns-5lqjm.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:16:52Z","stateTransitionTimes":{"ready":"2026-11-03T15:16:52Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:16:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:16:52Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:17:07.468Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"c7a30953-a962-4d09-b791-b6904b02a4f4","old JobStatus":"RUNNING","new JobStatus":"SUCCEEDED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-11-03T15:17:07.565Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"74297669-6cfe-423d-9f99-ef6ddc2cffda","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:17:07.565Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"74297669-6cfe-423d-9f99-ef6ddc2cffda","name":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:17:07.565Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"74297669-6cfe-423d-9f99-ef6ddc2cffda","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:17:07.565Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"74297669-6cfe-423d-9f99-ef6ddc2cffda"}
{"level":"info","ts":"2026-11-03T15:17:07.565Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"74297669-6cfe-423d-9f99-ef6ddc2cffda","rayCluster":"counter-8j287","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:17:07.568Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"74297669-6cfe-423d-9f99-ef6ddc2cffda","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:17:00Z","endTime":"2026-11-03T15:17:04Z"},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","dashboardURL":"counter-8j287-head-svc.test-ns-5lqjm.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:16:52Z","stateTransitionTimes":{"ready":"2026-11-03T15:16:52Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:16:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:16:52Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:17:00Z","endTime":"2026-11-03T15:17:04Z"},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","dashboardURL":"counter-8j287-head-svc.test-ns-5lqjm.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:16:52Z","stateTransitionTimes":{"ready":"2026-11-03T15:16:52Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:16:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:16:52Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:17:10.301Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"02e2240e-821c-4b78-8044-b87da437837e","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:17:10.301Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"02e2240e-821c-4b78-8044-b87da437837e","name":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:17:10.301Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"02e2240e-821c-4b78-8044-b87da437837e","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:17:10.301Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"02e2240e-821c-4b78-8044-b87da437837e"}
{"level":"info","ts":"2026-11-03T15:17:10.301Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"02e2240e-821c-4b78-8044-b87da437837e","rayCluster":"counter-8j287","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:17:10.304Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"02e2240e-821c-4b78-8044-b87da437837e","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:17:00Z","endTime":"2026-11-03T15:17:04Z"},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","dashboardURL":"counter-8j287-head-svc.test-ns-5lqjm.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:16:52Z","stateTransitionTimes":{"ready":"2026-11-03T15:16:52Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:16:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:16:52Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:17:00Z","endTime":"2026-11-03T15:17:04Z"},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","dashboardURL":"counter-8j287-head-svc.test-ns-5lqjm.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:16:52Z","stateTransitionTimes":{"ready":"2026-11-03T15:16:52Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:16:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:16:52Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:17:11.311Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"b8e16009-b920-4589-b02c-bd1d3f6759bf","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:17:11.311Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"b8e16009-b920-4589-b02c-bd1d3f6759bf","name":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:17:11.311Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"b8e16009-b920-4589-b02c-bd1d3f6759bf","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:17:11.311Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"b8e16009-b920-4589-b02c-bd1d3f6759bf"}
{"level":"info","ts":"2026-11-03T15:17:11.311Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"b8e16009-b920-4589-b02c-bd1d3f6759bf","rayCluster":"counter-8j287","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:17:11.314Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"b8e16009-b920-4589-b02c-bd1d3f6759bf","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:17:00Z","endTime":"2026-11-03T15:17:04Z"},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","dashboardURL":"counter-8j287-head-svc.test-ns-5lqjm.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:16:52Z","stateTransitionTimes":{"ready":"2026-11-03T15:16:52Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:16:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:16:52Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:17:00Z","endTime":"2026-11-03T15:17:04Z"},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","dashboardURL":"counter-8j287-head-svc.test-ns-5lqjm.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:16:52Z","stateTransitionTimes":{"ready":"2026-11-03T15:16:52Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:16:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:16:52Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:17:11.333Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"1f3b1afc-4ac8-4425-b1ed-e39beb19c7fc","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:17:11.333Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"1f3b1afc-4ac8-4425-b1ed-e39beb19c7fc","name":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:17:11.333Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"1f3b1afc-4ac8-4425-b1ed-e39beb19c7fc","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:17:11.333Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"1f3b1afc-4ac8-4425-b1ed-e39beb19c7fc"}
{"level":"info","ts":"2026-11-03T15:17:11.333Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"1f3b1afc-4ac8-4425-b1ed-e39beb19c7fc","rayCluster":"counter-8j287","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:17:11.336Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"1f3b1afc-4ac8-4425-b1ed-e39beb19c7fc","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:17:00Z","endTime":"2026-11-03T15:17:04Z"},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","dashboardURL":"counter-8j287-head-svc.test-ns-5lqjm.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-11-03T15:14:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:16:52Z","stateTransitionTimes":{"ready":"2026-11-03T15:16:52Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:16:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:16:52Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:17:00Z","endTime":"2026-11-03T15:17:04Z"},"jobId":"counter-p4vx4","rayClusterName":"counter-8j287","dashboardURL":"counter-8j287-head-svc.test-ns-5lqjm.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Complete","message":"Job finished successfully.","startTime":"2026-11-03T15:14:53Z","succeeded":1,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:16:52Z","stateTransitionTimes":{"ready":"2026-11-03T15:16:52Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.37","serviceIP":"10.132.0.37","podName":"counter-8j287-head-zldss","serviceName":"counter-8j287-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:16:36Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:16:52Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:14:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:14:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:17:11.336Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"1f3b1afc-4ac8-4425-b1ed-e39beb19c7fc","old JobStatus":"SUCCEEDED","new JobStatus":"SUCCEEDED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Complete"}
{"level":"info","ts":"2026-11-03T15:17:11.433Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"3f0e1089-8fa4-4dd1-b6d4-baeafc754dae","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:17:11.433Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"3f0e1089-8fa4-4dd1-b6d4-baeafc754dae","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Complete","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-11-03 15:17:11 +0000 UTC","Now":"2026-11-03T15:17:11.433Z","ShutdownTime":"2026-11-03T15:17:11.000Z"}
{"level":"info","ts":"2026-11-03T15:17:11.490Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob is deleted","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"3f0e1089-8fa4-4dd1-b6d4-baeafc754dae","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:17:11.490Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"3f0e1089-8fa4-4dd1-b6d4-baeafc754dae","isClusterDeleted":false}
{"level":"info","ts":"2026-11-03T15:17:11.490Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"3f0e1089-8fa4-4dd1-b6d4-baeafc754dae","deletionMechanism":"ShutdownAfterJobFinishes","RayCluster":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:17:11.491Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"counter-8j287","reconcileID":"1e075151-f840-4979-b03c-bb378ba5aae7","namespacedName":{"name":"counter-8j287","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:17:11.491Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"counter-8j287","reconcileID":"1e075151-f840-4979-b03c-bb378ba5aae7","cluster":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:17:11.491Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"counter-8j287","reconcileID":"1e075151-f840-4979-b03c-bb378ba5aae7","cluster":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:17:11.491Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"db5a720e-be82-4e05-acaa-5106c272befe","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:17:11.491Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"counter-8j287","reconcileID":"1e075151-f840-4979-b03c-bb378ba5aae7","namespace":"test-ns-5lqjm","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-11-03T15:17:11.491Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"db5a720e-be82-4e05-acaa-5106c272befe","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Complete","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-11-03 15:17:11 +0000 UTC","Now":"2026-11-03T15:17:11.491Z","ShutdownTime":"2026-11-03T15:17:11.000Z"}
{"level":"info","ts":"2026-11-03T15:17:11.491Z","logger":"controllers.RayJob","msg":"The deletion of the associated RayCluster for RayJob is ongoing.","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"db5a720e-be82-4e05-acaa-5106c272befe","RayCluster":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:17:11.491Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"db5a720e-be82-4e05-acaa-5106c272befe","isClusterDeleted":false}
{"level":"info","ts":"2026-11-03T15:17:11.491Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"db5a720e-be82-4e05-acaa-5106c272befe","deletionMechanism":"ShutdownAfterJobFinishes","RayCluster":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:17:11.491Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"3ec58334-564c-4090-8184-dc10373b36ce"}
{"level":"info","ts":"2026-11-03T15:17:11.543Z","logger":"authentication-controller","msg":"Deleted shared ReferenceGrant (last cluster with auth)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"counter-8j287","reconcileID":"1e075151-f840-4979-b03c-bb378ba5aae7","name":"kuberay-gateway-access","namespace":"test-ns-5lqjm"}
{"level":"info","ts":"2026-11-03T15:17:11.598Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"counter-8j287","reconcileID":"1e075151-f840-4979-b03c-bb378ba5aae7","name":"test-ns-5lqjm-counter-8j287","namespace":"opendatahub"}
{"level":"info","ts":"2026-11-03T15:17:11.652Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"counter-8j287","reconcileID":"1e075151-f840-4979-b03c-bb378ba5aae7","configMap":"kube-rbac-proxy-config-counter-8j287"}
{"level":"info","ts":"2026-11-03T15:17:11.704Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"counter-8j287","reconcileID":"1e075151-f840-4979-b03c-bb378ba5aae7","serviceAccount":"counter-8j287-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-11-03T15:17:11.881Z","logger":"raycluster-resource","msg":"default","name":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:17:11.881Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"counter-8j287","namespace":"test-ns-5lqjm"}
{"level":"info","ts":"2026-11-03T15:17:12.030Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"d83fc8c9-6d20-4a4f-b8e2-306e7e17b72b","DeletionTimestamp":"2026-11-03 15:17:11 +0000 UTC"}
{"level":"info","ts":"2026-11-03T15:17:12.030Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"d83fc8c9-6d20-4a4f-b8e2-306e7e17b72b","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-11-03T15:17:12.045Z","logger":"raycluster-resource","msg":"validate update","name":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:17:12.138Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"ea2d6a31-5e62-4953-96e8-9917ee2a4932","DeletionTimestamp":"2026-11-03 15:17:11 +0000 UTC"}
{"level":"info","ts":"2026-11-03T15:17:12.138Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"ea2d6a31-5e62-4953-96e8-9917ee2a4932","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-11-03T15:17:12.140Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"reconcileID":"7ad63eaa-de41-4ba2-bf7e-d6e57c9a8184"}
{"level":"info","ts":"2026-11-03T15:17:12.141Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"counter-8j287","reconcileID":"1e075151-f840-4979-b03c-bb378ba5aae7","cluster":"counter-8j287"}
{"level":"info","ts":"2026-11-03T15:17:12.141Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"counter-8j287","reconcileID":"d1a57401-b92e-4460-a0b7-98dec92fa36c","namespacedName":{"name":"counter-8j287","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:17:12.141Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"counter-8j287","reconcileID":"d1a57401-b92e-4460-a0b7-98dec92fa36c"}
{"level":"info","ts":"2026-11-03T15:17:12.141Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"counter-8j287","reconcileID":"d1a57401-b92e-4460-a0b7-98dec92fa36c","namespace":"test-ns-5lqjm","authClusters":0}
{"level":"info","ts":"2026-11-03T15:17:12.141Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"counter-8j287","reconcileID":"d1a57401-b92e-4460-a0b7-98dec92fa36c","namespace":"test-ns-5lqjm"}
{"level":"info","ts":"2026-11-03T15:17:12.141Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"counter-8j287","reconcileID":"c281c13e-3363-46b7-97e6-7888b82cf122","namespacedName":{"name":"counter-8j287","namespace":"test-ns-5lqjm"}}
{"level":"info","ts":"2026-11-03T15:17:12.141Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"counter-8j287","reconcileID":"c281c13e-3363-46b7-97e6-7888b82cf122"}
{"level":"info","ts":"2026-11-03T15:17:12.141Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"counter-8j287","reconcileID":"c281c13e-3363-46b7-97e6-7888b82cf122","namespace":"test-ns-5lqjm","authClusters":0}
{"level":"info","ts":"2026-11-03T15:17:12.141Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-8j287","namespace":"test-ns-5lqjm"},"namespace":"test-ns-5lqjm","name":"counter-8j287","reconcileID":"c281c13e-3363-46b7-97e6-7888b82cf122","namespace":"test-ns-5lqjm"}
{"level":"error","ts":"2026-11-03T15:17:12.235Z","logger":"controllers.RayJob","msg":"Failed to remove finalizer for RayJob","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"ea2d6a31-5e62-4953-96e8-9917ee2a4932","error":"rayjobs.ray.io \"counter\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:135\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:17:12.235Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"ea2d6a31-5e62-4953-96e8-9917ee2a4932"}
{"level":"error","ts":"2026-11-03T15:17:12.235Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"ea2d6a31-5e62-4953-96e8-9917ee2a4932","error":"rayjobs.ray.io \"counter\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:17:12.235Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"counter","namespace":"test-ns-5lqjm"},"reconcileID":"15a2c035-fcdb-48f3-bc60-7b831179eb69"}
{"level":"info","ts":"2026-11-03T15:17:15.286Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"275b1054-231a-4153-bb76-4740eeaf0f26","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:17:15.286Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"275b1054-231a-4153-bb76-4740eeaf0f26","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-11-03T15:17:15.345Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"275b1054-231a-4153-bb76-4740eeaf0f26"}
{"level":"info","ts":"2026-11-03T15:17:15.345Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"275b1054-231a-4153-bb76-4740eeaf0f26","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-11-03T15:17:15.345Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"275b1054-231a-4153-bb76-4740eeaf0f26","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:17:15.345Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"275b1054-231a-4153-bb76-4740eeaf0f26","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:17:15.403Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"9baa09f5-afe1-450f-9d14-127204c5c7e4","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:17:15.403Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"9baa09f5-afe1-450f-9d14-127204c5c7e4","name":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:17:15.403Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"9baa09f5-afe1-450f-9d14-127204c5c7e4","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:17:15.403Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"9baa09f5-afe1-450f-9d14-127204c5c7e4","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:17:15.453Z","logger":"raycluster-resource","msg":"default","name":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:17:15.453Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"counter-n8dh7","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:17:15.512Z","logger":"raycluster-resource","msg":"validate create","name":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:17:15.565Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"counter-n8dh7","reconcileID":"52d0091b-6965-4d71-b131-db053ab6c68f","namespacedName":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:17:15.565Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"counter-n8dh7","reconcileID":"52d0091b-6965-4d71-b131-db053ab6c68f","mode":"IntegratedOAuth","cluster":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:17:15.565Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"counter-n8dh7","reconcileID":"52d0091b-6965-4d71-b131-db053ab6c68f","cluster":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:17:15.565Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"counter-n8dh7","reconcileID":"52d0091b-6965-4d71-b131-db053ab6c68f","cluster":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:17:15.566Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"a679cadb-2cb1-4fbd-a3d7-ead07eb421e0"}
{"level":"info","ts":"2026-11-03T15:17:15.566Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"a679cadb-2cb1-4fbd-a3d7-ead07eb421e0"}
{"level":"info","ts":"2026-11-03T15:17:15.566Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"aed70dfb-4983-4c4f-afb4-524c40abcfda","cluster":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:17:15.565Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"9baa09f5-afe1-450f-9d14-127204c5c7e4","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:17:15.566Z","logger":"controllers.NetworkPolicy","msg":"Reconciling NetworkPolicies for RayCluster","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"1851a612-1a32-4fe9-baad-7820fd2f9d0b","cluster":"counter-n8dh7","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:17:15.566Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"9baa09f5-afe1-450f-9d14-127204c5c7e4"}
{"level":"info","ts":"2026-11-03T15:17:15.566Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"9baa09f5-afe1-450f-9d14-127204c5c7e4","RayCluster":"counter-n8dh7","State":""}
{"level":"info","ts":"2026-11-03T15:17:15.566Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"9baa09f5-afe1-450f-9d14-127204c5c7e4","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:17:15.566Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"dcfef54d-c70f-4b37-a701-360a652e9a15","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:17:15.566Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"dcfef54d-c70f-4b37-a701-360a652e9a15","name":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:17:15.566Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"dcfef54d-c70f-4b37-a701-360a652e9a15","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:17:15.566Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"dcfef54d-c70f-4b37-a701-360a652e9a15"}
{"level":"info","ts":"2026-11-03T15:17:15.566Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"dcfef54d-c70f-4b37-a701-360a652e9a15","RayCluster":"counter-n8dh7","State":""}
{"level":"info","ts":"2026-11-03T15:17:15.566Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"dcfef54d-c70f-4b37-a701-360a652e9a15","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:17:15.617Z","logger":"raycluster-resource","msg":"default","name":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:17:15.617Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"counter-n8dh7","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:17:15.618Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"a679cadb-2cb1-4fbd-a3d7-ead07eb421e0","name":"counter-n8dh7-head-svc"}
{"level":"info","ts":"2026-11-03T15:17:15.618Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"a679cadb-2cb1-4fbd-a3d7-ead07eb421e0"}
{"level":"info","ts":"2026-11-03T15:17:15.664Z","logger":"controllers.NetworkPolicy","msg":"Successfully created NetworkPolicy","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"1851a612-1a32-4fe9-baad-7820fd2f9d0b","name":"counter-n8dh7-head"}
{"level":"info","ts":"2026-11-03T15:17:15.668Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"a679cadb-2cb1-4fbd-a3d7-ead07eb421e0","error":"certificate ray-head-cert-counter-n8dh7 not found: certificates.cert-manager.io \"ray-head-cert-counter-n8dh7\" not found"}
{"level":"error","ts":"2026-11-03T15:17:15.668Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"a679cadb-2cb1-4fbd-a3d7-ead07eb421e0","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-counter-n8dh7 not found: certificates.cert-manager.io \"ray-head-cert-counter-n8dh7\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:17:15.668Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"a679cadb-2cb1-4fbd-a3d7-ead07eb421e0","error":"mTLS secrets not ready: certificate ray-head-cert-counter-n8dh7 not found: certificates.cert-manager.io \"ray-head-cert-counter-n8dh7\" not found"}
{"level":"info","ts":"2026-11-03T15:17:15.668Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"a679cadb-2cb1-4fbd-a3d7-ead07eb421e0","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"counter-n8dh7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:17:15.668Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"a679cadb-2cb1-4fbd-a3d7-ead07eb421e0","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"counter-n8dh7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:17:15.668Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"a679cadb-2cb1-4fbd-a3d7-ead07eb421e0","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"counter-n8dh7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:17:15.668Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"a679cadb-2cb1-4fbd-a3d7-ead07eb421e0","name":"counter-n8dh7","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:17:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:17:15.673Z","logger":"raycluster-resource","msg":"validate update","name":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:17:15.717Z","logger":"controllers.NetworkPolicy","msg":"Successfully created NetworkPolicy","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"1851a612-1a32-4fe9-baad-7820fd2f9d0b","name":"counter-n8dh7-workers-small-group"}
{"level":"info","ts":"2026-11-03T15:17:15.717Z","logger":"controllers.NetworkPolicy","msg":"Successfully reconciled NetworkPolicies for RayCluster","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"1851a612-1a32-4fe9-baad-7820fd2f9d0b","cluster":"counter-n8dh7","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:17:15.717Z","logger":"controllers.NetworkPolicy","msg":"Reconciling NetworkPolicies for RayCluster","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"6b5f77b6-470d-4d4d-bd33-99a5630165f9","cluster":"counter-n8dh7","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:17:15.727Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"9f1dc4dc-37cd-42f1-b0a8-bb6f8c165fea","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:17:15.727Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"9f1dc4dc-37cd-42f1-b0a8-bb6f8c165fea","name":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:17:15.727Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"9f1dc4dc-37cd-42f1-b0a8-bb6f8c165fea","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:17:15.728Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"9f1dc4dc-37cd-42f1-b0a8-bb6f8c165fea"}
{"level":"info","ts":"2026-11-03T15:17:15.728Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"9f1dc4dc-37cd-42f1-b0a8-bb6f8c165fea","RayCluster":"counter-n8dh7","State":""}
{"level":"info","ts":"2026-11-03T15:17:15.728Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"9f1dc4dc-37cd-42f1-b0a8-bb6f8c165fea","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:17:15.731Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"a679cadb-2cb1-4fbd-a3d7-ead07eb421e0","name":"counter-n8dh7","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-n8dh7\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-kbx4w","name":"counter-n8dh7"}}
{"level":"info","ts":"2026-11-03T15:17:15.731Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"a679cadb-2cb1-4fbd-a3d7-ead07eb421e0"}
{"level":"error","ts":"2026-11-03T15:17:15.731Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"a679cadb-2cb1-4fbd-a3d7-ead07eb421e0","error":"mTLS secrets not ready: certificate ray-head-cert-counter-n8dh7 not found: certificates.cert-manager.io \"ray-head-cert-counter-n8dh7\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:17:15.732Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"26bab303-1e55-4723-8c80-89f1cf9f7dae"}
{"level":"info","ts":"2026-11-03T15:17:15.732Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"26bab303-1e55-4723-8c80-89f1cf9f7dae"}
{"level":"info","ts":"2026-11-03T15:17:15.732Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"26bab303-1e55-4723-8c80-89f1cf9f7dae","1 head service found":"counter-n8dh7-head-svc"}
{"level":"info","ts":"2026-11-03T15:17:15.732Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"26bab303-1e55-4723-8c80-89f1cf9f7dae"}
{"level":"info","ts":"2026-11-03T15:17:15.779Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"counter-n8dh7","reconcileID":"52d0091b-6965-4d71-b131-db053ab6c68f","name":"counter-n8dh7-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-11-03T15:17:15.781Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"26bab303-1e55-4723-8c80-89f1cf9f7dae","error":"certificate ray-head-cert-counter-n8dh7 not found: certificates.cert-manager.io \"ray-head-cert-counter-n8dh7\" not found"}
{"level":"error","ts":"2026-11-03T15:17:15.781Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"26bab303-1e55-4723-8c80-89f1cf9f7dae","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-counter-n8dh7 not found: certificates.cert-manager.io \"ray-head-cert-counter-n8dh7\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:17:15.781Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"26bab303-1e55-4723-8c80-89f1cf9f7dae","error":"mTLS secrets not ready: certificate ray-head-cert-counter-n8dh7 not found: certificates.cert-manager.io \"ray-head-cert-counter-n8dh7\" not found"}
{"level":"info","ts":"2026-11-03T15:17:15.781Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"26bab303-1e55-4723-8c80-89f1cf9f7dae","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"counter-n8dh7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:17:15.781Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"26bab303-1e55-4723-8c80-89f1cf9f7dae","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"counter-n8dh7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:17:15.781Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"26bab303-1e55-4723-8c80-89f1cf9f7dae","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"counter-n8dh7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:17:15.781Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"26bab303-1e55-4723-8c80-89f1cf9f7dae","name":"counter-n8dh7","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:17:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:17:15.827Z","logger":"controllers.NetworkPolicy","msg":"Successfully reconciled NetworkPolicies for RayCluster","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"6b5f77b6-470d-4d4d-bd33-99a5630165f9","cluster":"counter-n8dh7","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:17:15.834Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"counter-n8dh7","reconcileID":"52d0091b-6965-4d71-b131-db053ab6c68f","name":"kuberay-gateway-access","namespace":"test-ns-kbx4w","operation":"created"}
{"level":"info","ts":"2026-11-03T15:17:15.852Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"26bab303-1e55-4723-8c80-89f1cf9f7dae"}
{"level":"error","ts":"2026-11-03T15:17:15.852Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"26bab303-1e55-4723-8c80-89f1cf9f7dae","error":"mTLS secrets not ready: certificate ray-head-cert-counter-n8dh7 not found: certificates.cert-manager.io \"ray-head-cert-counter-n8dh7\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:17:15.852Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"06fb2ce8-1df2-4397-96b6-238c309af686","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:17:15.852Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"06fb2ce8-1df2-4397-96b6-238c309af686","name":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:17:15.852Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"06fb2ce8-1df2-4397-96b6-238c309af686","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:17:15.852Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"06fb2ce8-1df2-4397-96b6-238c309af686"}
{"level":"info","ts":"2026-11-03T15:17:15.852Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"06fb2ce8-1df2-4397-96b6-238c309af686","RayCluster":"counter-n8dh7","State":""}
{"level":"info","ts":"2026-11-03T15:17:15.852Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"06fb2ce8-1df2-4397-96b6-238c309af686","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:17:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:17:15.853Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"06fb2ce8-1df2-4397-96b6-238c309af686","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:17:15.862Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"077aad51-208c-4374-9b8f-9e82b6eab01a"}
{"level":"info","ts":"2026-11-03T15:17:15.862Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"077aad51-208c-4374-9b8f-9e82b6eab01a"}
{"level":"info","ts":"2026-11-03T15:17:15.862Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"077aad51-208c-4374-9b8f-9e82b6eab01a","1 head service found":"counter-n8dh7-head-svc"}
{"level":"info","ts":"2026-11-03T15:17:15.862Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"077aad51-208c-4374-9b8f-9e82b6eab01a"}
{"level":"info","ts":"2026-11-03T15:17:15.889Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"counter-n8dh7","reconcileID":"52d0091b-6965-4d71-b131-db053ab6c68f","name":"test-ns-kbx4w-counter-n8dh7","operation":"created"}
{"level":"info","ts":"2026-11-03T15:17:15.930Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"077aad51-208c-4374-9b8f-9e82b6eab01a","error":"certificate ray-head-cert-counter-n8dh7 not found: certificates.cert-manager.io \"ray-head-cert-counter-n8dh7\" not found"}
{"level":"error","ts":"2026-11-03T15:17:15.930Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"077aad51-208c-4374-9b8f-9e82b6eab01a","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-counter-n8dh7 not found: certificates.cert-manager.io \"ray-head-cert-counter-n8dh7\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:17:15.930Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"077aad51-208c-4374-9b8f-9e82b6eab01a","error":"mTLS secrets not ready: certificate ray-head-cert-counter-n8dh7 not found: certificates.cert-manager.io \"ray-head-cert-counter-n8dh7\" not found"}
{"level":"info","ts":"2026-11-03T15:17:15.930Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"077aad51-208c-4374-9b8f-9e82b6eab01a","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"counter-n8dh7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:17:15.930Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"077aad51-208c-4374-9b8f-9e82b6eab01a","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"counter-n8dh7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:17:15.930Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"077aad51-208c-4374-9b8f-9e82b6eab01a","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"counter-n8dh7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:17:15.930Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"077aad51-208c-4374-9b8f-9e82b6eab01a"}
{"level":"error","ts":"2026-11-03T15:17:15.930Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"077aad51-208c-4374-9b8f-9e82b6eab01a","error":"mTLS secrets not ready: certificate ray-head-cert-counter-n8dh7 not found: certificates.cert-manager.io \"ray-head-cert-counter-n8dh7\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:17:15.943Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"counter-n8dh7","reconcileID":"52d0091b-6965-4d71-b131-db053ab6c68f","name":"kube-rbac-proxy-config-counter-n8dh7","operation":"created"}
{"level":"info","ts":"2026-11-03T15:17:15.945Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"6d7db59c-e9bb-4191-9611-eb6018e39700","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:17:15.945Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"6d7db59c-e9bb-4191-9611-eb6018e39700","name":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:17:15.945Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"6d7db59c-e9bb-4191-9611-eb6018e39700","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:17:15.945Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"6d7db59c-e9bb-4191-9611-eb6018e39700"}
{"level":"info","ts":"2026-11-03T15:17:15.945Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"6d7db59c-e9bb-4191-9611-eb6018e39700","RayCluster":"counter-n8dh7","State":""}
{"level":"info","ts":"2026-11-03T15:17:15.945Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"6d7db59c-e9bb-4191-9611-eb6018e39700","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:17:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:17:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:17:15.951Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"4870d43e-c88f-459d-b1bc-9b2cd83be4a5"}
{"level":"info","ts":"2026-11-03T15:17:15.951Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"4870d43e-c88f-459d-b1bc-9b2cd83be4a5"}
{"level":"info","ts":"2026-11-03T15:17:15.951Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"4870d43e-c88f-459d-b1bc-9b2cd83be4a5","1 head service found":"counter-n8dh7-head-svc"}
{"level":"info","ts":"2026-11-03T15:17:15.951Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"4870d43e-c88f-459d-b1bc-9b2cd83be4a5"}
{"level":"error","ts":"2026-11-03T15:17:15.997Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"counter-n8dh7","reconcileID":"52d0091b-6965-4d71-b131-db053ab6c68f","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"counter-n8dh7\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:167\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:17:15.997Z","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"counter-n8dh7","reconcileID":"52d0091b-6965-4d71-b131-db053ab6c68f"}
{"level":"error","ts":"2026-11-03T15:17:15.997Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"counter-n8dh7","reconcileID":"52d0091b-6965-4d71-b131-db053ab6c68f","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"counter-n8dh7\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:17:15.997Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"counter-n8dh7","reconcileID":"b7b9be9c-799c-4279-be17-faa8c471080b","namespacedName":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:17:15.997Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"counter-n8dh7","reconcileID":"b7b9be9c-799c-4279-be17-faa8c471080b","mode":"IntegratedOAuth","cluster":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:17:15.997Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"counter-n8dh7","reconcileID":"b7b9be9c-799c-4279-be17-faa8c471080b","cluster":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:17:16.000Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"4870d43e-c88f-459d-b1bc-9b2cd83be4a5","error":"certificate ray-head-cert-counter-n8dh7 not found: certificates.cert-manager.io \"ray-head-cert-counter-n8dh7\" not found"}
{"level":"error","ts":"2026-11-03T15:17:16.000Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"4870d43e-c88f-459d-b1bc-9b2cd83be4a5","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-counter-n8dh7 not found: certificates.cert-manager.io \"ray-head-cert-counter-n8dh7\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:17:16.001Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"4870d43e-c88f-459d-b1bc-9b2cd83be4a5","error":"mTLS secrets not ready: certificate ray-head-cert-counter-n8dh7 not found: certificates.cert-manager.io \"ray-head-cert-counter-n8dh7\" not found"}
{"level":"info","ts":"2026-11-03T15:17:16.001Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"4870d43e-c88f-459d-b1bc-9b2cd83be4a5","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"counter-n8dh7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:17:16.001Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"4870d43e-c88f-459d-b1bc-9b2cd83be4a5","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"counter-n8dh7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:17:16.001Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"4870d43e-c88f-459d-b1bc-9b2cd83be4a5","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"counter-n8dh7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:17:16.001Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"4870d43e-c88f-459d-b1bc-9b2cd83be4a5"}
{"level":"error","ts":"2026-11-03T15:17:16.001Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"4870d43e-c88f-459d-b1bc-9b2cd83be4a5","error":"mTLS secrets not ready: certificate ray-head-cert-counter-n8dh7 not found: certificates.cert-manager.io \"ray-head-cert-counter-n8dh7\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:17:16.042Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"cb480f8e-91b7-4f25-b796-50df8ee02aa8"}
{"level":"info","ts":"2026-11-03T15:17:16.042Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"cb480f8e-91b7-4f25-b796-50df8ee02aa8"}
{"level":"info","ts":"2026-11-03T15:17:16.042Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"cb480f8e-91b7-4f25-b796-50df8ee02aa8","1 head service found":"counter-n8dh7-head-svc"}
{"level":"info","ts":"2026-11-03T15:17:16.042Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"cb480f8e-91b7-4f25-b796-50df8ee02aa8"}
{"level":"info","ts":"2026-11-03T15:17:16.061Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"counter-n8dh7","reconcileID":"b7b9be9c-799c-4279-be17-faa8c471080b","cluster":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:17:16.061Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"547505c3-5dca-433b-bd7d-b6f0f1d25e94","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:17:16.061Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"547505c3-5dca-433b-bd7d-b6f0f1d25e94","name":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:17:16.061Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"547505c3-5dca-433b-bd7d-b6f0f1d25e94","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:17:16.062Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"547505c3-5dca-433b-bd7d-b6f0f1d25e94"}
{"level":"info","ts":"2026-11-03T15:17:16.062Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"547505c3-5dca-433b-bd7d-b6f0f1d25e94","RayCluster":"counter-n8dh7","State":""}
{"level":"info","ts":"2026-11-03T15:17:16.062Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"547505c3-5dca-433b-bd7d-b6f0f1d25e94","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:17:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:17:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:17:16.062Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"547505c3-5dca-433b-bd7d-b6f0f1d25e94","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:17:16.091Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"cb480f8e-91b7-4f25-b796-50df8ee02aa8","error":"certificate ray-head-cert-counter-n8dh7 not found: certificates.cert-manager.io \"ray-head-cert-counter-n8dh7\" not found"}
{"level":"error","ts":"2026-11-03T15:17:16.091Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"cb480f8e-91b7-4f25-b796-50df8ee02aa8","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-counter-n8dh7 not found: certificates.cert-manager.io \"ray-head-cert-counter-n8dh7\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:17:16.091Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"cb480f8e-91b7-4f25-b796-50df8ee02aa8","error":"mTLS secrets not ready: certificate ray-head-cert-counter-n8dh7 not found: certificates.cert-manager.io \"ray-head-cert-counter-n8dh7\" not found"}
{"level":"info","ts":"2026-11-03T15:17:16.091Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"cb480f8e-91b7-4f25-b796-50df8ee02aa8","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"counter-n8dh7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:17:16.091Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"cb480f8e-91b7-4f25-b796-50df8ee02aa8","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"counter-n8dh7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:17:16.091Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"cb480f8e-91b7-4f25-b796-50df8ee02aa8","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"counter-n8dh7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:17:16.091Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"cb480f8e-91b7-4f25-b796-50df8ee02aa8"}
{"level":"error","ts":"2026-11-03T15:17:16.091Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"cb480f8e-91b7-4f25-b796-50df8ee02aa8","error":"mTLS secrets not ready: certificate ray-head-cert-counter-n8dh7 not found: certificates.cert-manager.io \"ray-head-cert-counter-n8dh7\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:17:16.143Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"9b18579a-b1e6-4d5a-a2fc-63faaa0c08ea","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:17:16.143Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"9b18579a-b1e6-4d5a-a2fc-63faaa0c08ea","name":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:17:16.143Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"9b18579a-b1e6-4d5a-a2fc-63faaa0c08ea","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:17:16.143Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"9b18579a-b1e6-4d5a-a2fc-63faaa0c08ea"}
{"level":"info","ts":"2026-11-03T15:17:16.143Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"9b18579a-b1e6-4d5a-a2fc-63faaa0c08ea","RayCluster":"counter-n8dh7","State":""}
{"level":"info","ts":"2026-11-03T15:17:16.143Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"9b18579a-b1e6-4d5a-a2fc-63faaa0c08ea","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:17:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:17:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:17:16.172Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"a0d1fb97-f97b-41f0-9505-734a2593a6f2"}
{"level":"info","ts":"2026-11-03T15:17:16.172Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"a0d1fb97-f97b-41f0-9505-734a2593a6f2"}
{"level":"info","ts":"2026-11-03T15:17:16.172Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"a0d1fb97-f97b-41f0-9505-734a2593a6f2","1 head service found":"counter-n8dh7-head-svc"}
{"level":"info","ts":"2026-11-03T15:17:16.172Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"a0d1fb97-f97b-41f0-9505-734a2593a6f2"}
{"level":"info","ts":"2026-11-03T15:17:16.221Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"a0d1fb97-f97b-41f0-9505-734a2593a6f2","error":"certificate ray-head-cert-counter-n8dh7 not found: certificates.cert-manager.io \"ray-head-cert-counter-n8dh7\" not found"}
{"level":"error","ts":"2026-11-03T15:17:16.221Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"a0d1fb97-f97b-41f0-9505-734a2593a6f2","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-counter-n8dh7 not found: certificates.cert-manager.io \"ray-head-cert-counter-n8dh7\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:17:16.222Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"a0d1fb97-f97b-41f0-9505-734a2593a6f2","error":"mTLS secrets not ready: certificate ray-head-cert-counter-n8dh7 not found: certificates.cert-manager.io \"ray-head-cert-counter-n8dh7\" not found"}
{"level":"info","ts":"2026-11-03T15:17:16.222Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"a0d1fb97-f97b-41f0-9505-734a2593a6f2","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"counter-n8dh7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:17:16.222Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"a0d1fb97-f97b-41f0-9505-734a2593a6f2","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"counter-n8dh7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:17:16.222Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"a0d1fb97-f97b-41f0-9505-734a2593a6f2","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"counter-n8dh7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:17:16.222Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"a0d1fb97-f97b-41f0-9505-734a2593a6f2"}
{"level":"error","ts":"2026-11-03T15:17:16.222Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"a0d1fb97-f97b-41f0-9505-734a2593a6f2","error":"mTLS secrets not ready: certificate ray-head-cert-counter-n8dh7 not found: certificates.cert-manager.io \"ray-head-cert-counter-n8dh7\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:17:16.383Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"3e9cd198-7a4d-47c3-baeb-f7cb45c25b52"}
{"level":"info","ts":"2026-11-03T15:17:16.383Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"3e9cd198-7a4d-47c3-baeb-f7cb45c25b52"}
{"level":"info","ts":"2026-11-03T15:17:16.383Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"3e9cd198-7a4d-47c3-baeb-f7cb45c25b52","1 head service found":"counter-n8dh7-head-svc"}
{"level":"info","ts":"2026-11-03T15:17:16.383Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"3e9cd198-7a4d-47c3-baeb-f7cb45c25b52"}
{"level":"info","ts":"2026-11-03T15:17:16.433Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"3e9cd198-7a4d-47c3-baeb-f7cb45c25b52","error":"certificate ray-head-cert-counter-n8dh7 is not ready at current generation 1"}
{"level":"error","ts":"2026-11-03T15:17:16.433Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"3e9cd198-7a4d-47c3-baeb-f7cb45c25b52","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-counter-n8dh7 is not ready at current generation 1","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:17:16.433Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"3e9cd198-7a4d-47c3-baeb-f7cb45c25b52","error":"mTLS secrets not ready: certificate ray-head-cert-counter-n8dh7 is not ready at current generation 1"}
{"level":"info","ts":"2026-11-03T15:17:16.433Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"3e9cd198-7a4d-47c3-baeb-f7cb45c25b52","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"counter-n8dh7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:17:16.433Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"3e9cd198-7a4d-47c3-baeb-f7cb45c25b52","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"counter-n8dh7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:17:16.433Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"3e9cd198-7a4d-47c3-baeb-f7cb45c25b52","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"counter-n8dh7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:17:16.433Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"3e9cd198-7a4d-47c3-baeb-f7cb45c25b52"}
{"level":"error","ts":"2026-11-03T15:17:16.433Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"3e9cd198-7a4d-47c3-baeb-f7cb45c25b52","error":"mTLS secrets not ready: certificate ray-head-cert-counter-n8dh7 is not ready at current generation 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:17:16.537Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"aed70dfb-4983-4c4f-afb4-524c40abcfda","certificate":"ray-head-cert-counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:17:16.537Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"aed70dfb-4983-4c4f-afb4-524c40abcfda"}
{"level":"info","ts":"2026-11-03T15:17:16.537Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"0281aee9-8500-4a60-9799-5c20c2b71eeb","cluster":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:17:16.754Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"b4e7c980-5a1d-4744-b6e3-f3e733dd6b69"}
{"level":"info","ts":"2026-11-03T15:17:16.754Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"b4e7c980-5a1d-4744-b6e3-f3e733dd6b69"}
{"level":"info","ts":"2026-11-03T15:17:16.754Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"b4e7c980-5a1d-4744-b6e3-f3e733dd6b69","1 head service found":"counter-n8dh7-head-svc"}
{"level":"info","ts":"2026-11-03T15:17:16.754Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"b4e7c980-5a1d-4744-b6e3-f3e733dd6b69"}
{"level":"info","ts":"2026-11-03T15:17:16.805Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"b4e7c980-5a1d-4744-b6e3-f3e733dd6b69","error":"certificate ray-head-cert-counter-n8dh7 is not ready at current generation 1"}
{"level":"error","ts":"2026-11-03T15:17:16.805Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"b4e7c980-5a1d-4744-b6e3-f3e733dd6b69","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-counter-n8dh7 is not ready at current generation 1","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:17:16.805Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"b4e7c980-5a1d-4744-b6e3-f3e733dd6b69","error":"mTLS secrets not ready: certificate ray-head-cert-counter-n8dh7 is not ready at current generation 1"}
{"level":"info","ts":"2026-11-03T15:17:16.805Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"b4e7c980-5a1d-4744-b6e3-f3e733dd6b69","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"counter-n8dh7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:17:16.805Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"b4e7c980-5a1d-4744-b6e3-f3e733dd6b69","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"counter-n8dh7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:17:16.805Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"b4e7c980-5a1d-4744-b6e3-f3e733dd6b69","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"counter-n8dh7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:17:16.805Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"b4e7c980-5a1d-4744-b6e3-f3e733dd6b69"}
{"level":"error","ts":"2026-11-03T15:17:16.805Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"b4e7c980-5a1d-4744-b6e3-f3e733dd6b69","error":"mTLS secrets not ready: certificate ray-head-cert-counter-n8dh7 is not ready at current generation 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:17:16.988Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"0281aee9-8500-4a60-9799-5c20c2b71eeb","certificate":"ray-head-cert-counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:17:16.988Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"0281aee9-8500-4a60-9799-5c20c2b71eeb"}
{"level":"info","ts":"2026-11-03T15:17:17.446Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"246f0412-0cba-4329-bef8-61968605ad86"}
{"level":"info","ts":"2026-11-03T15:17:17.446Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"246f0412-0cba-4329-bef8-61968605ad86"}
{"level":"info","ts":"2026-11-03T15:17:17.446Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"246f0412-0cba-4329-bef8-61968605ad86","1 head service found":"counter-n8dh7-head-svc"}
{"level":"info","ts":"2026-11-03T15:17:17.446Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"246f0412-0cba-4329-bef8-61968605ad86"}
{"level":"info","ts":"2026-11-03T15:17:17.497Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"246f0412-0cba-4329-bef8-61968605ad86","error":"certificate ray-head-cert-counter-n8dh7 is not ready at current generation 1"}
{"level":"error","ts":"2026-11-03T15:17:17.497Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"246f0412-0cba-4329-bef8-61968605ad86","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-counter-n8dh7 is not ready at current generation 1","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:17:17.497Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"246f0412-0cba-4329-bef8-61968605ad86","error":"mTLS secrets not ready: certificate ray-head-cert-counter-n8dh7 is not ready at current generation 1"}
{"level":"info","ts":"2026-11-03T15:17:17.497Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"246f0412-0cba-4329-bef8-61968605ad86","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"counter-n8dh7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:17:17.497Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"246f0412-0cba-4329-bef8-61968605ad86","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"counter-n8dh7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:17:17.497Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"246f0412-0cba-4329-bef8-61968605ad86","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"counter-n8dh7","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:17:17.497Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"246f0412-0cba-4329-bef8-61968605ad86"}
{"level":"error","ts":"2026-11-03T15:17:17.497Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"246f0412-0cba-4329-bef8-61968605ad86","error":"mTLS secrets not ready: certificate ray-head-cert-counter-n8dh7 is not ready at current generation 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:17:18.778Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"1ef0946e-4986-42c4-919d-7e4ec9f4c6a6"}
{"level":"info","ts":"2026-11-03T15:17:18.778Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"1ef0946e-4986-42c4-919d-7e4ec9f4c6a6"}
{"level":"info","ts":"2026-11-03T15:17:18.778Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"1ef0946e-4986-42c4-919d-7e4ec9f4c6a6","1 head service found":"counter-n8dh7-head-svc"}
{"level":"info","ts":"2026-11-03T15:17:18.778Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"1ef0946e-4986-42c4-919d-7e4ec9f4c6a6"}
{"level":"info","ts":"2026-11-03T15:17:18.879Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"1ef0946e-4986-42c4-919d-7e4ec9f4c6a6","mode":"IntegratedOAuth","cluster":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:17:18.879Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"1ef0946e-4986-42c4-919d-7e4ec9f4c6a6","cluster":"counter-n8dh7","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-11-03T15:17:18.879Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"1ef0946e-4986-42c4-919d-7e4ec9f4c6a6","cluster":"counter-n8dh7","authType":"IntegratedOAuth","serviceAccount":"counter-n8dh7-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-11-03T15:17:18.879Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"1ef0946e-4986-42c4-919d-7e4ec9f4c6a6","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"counter-n8dh7","ray.io/group":"headgroup","ray.io/identifier":"counter-n8dh7-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-11-03T15:17:18.879Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"1ef0946e-4986-42c4-919d-7e4ec9f4c6a6","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-11-03T15:17:18.879Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"1ef0946e-4986-42c4-919d-7e4ec9f4c6a6","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-11-03T15:17:18.879Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"1ef0946e-4986-42c4-919d-7e4ec9f4c6a6","rayNodeType":"head","generatedCmd":"ulimit -n ${RAY_START_ULIMIT_OPEN_FILES:-65536}; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-11-03T15:17:18.879Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"1ef0946e-4986-42c4-919d-7e4ec9f4c6a6","enabled":true}
{"level":"info","ts":"2026-11-03T15:17:18.943Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"1ef0946e-4986-42c4-919d-7e4ec9f4c6a6","name":"counter-n8dh7-head-cshcl"}
{"level":"info","ts":"2026-11-03T15:17:18.943Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"1ef0946e-4986-42c4-919d-7e4ec9f4c6a6","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:17:18.943Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"1ef0946e-4986-42c4-919d-7e4ec9f4c6a6","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:17:18.943Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"1ef0946e-4986-42c4-919d-7e4ec9f4c6a6","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-11-03T15:17:18.943Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"1ef0946e-4986-42c4-919d-7e4ec9f4c6a6","group":"small-group","indices":{}}
{"level":"info","ts":"2026-11-03T15:17:18.943Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"1ef0946e-4986-42c4-919d-7e4ec9f4c6a6","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-11-03T15:17:19.044Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"1ef0946e-4986-42c4-919d-7e4ec9f4c6a6","creating worker for group":"small-group","index":0,"total":1,"replicaIndex":0}
{"level":"info","ts":"2026-11-03T15:17:19.044Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"1ef0946e-4986-42c4-919d-7e4ec9f4c6a6","nodeType":"worker","rayStartParams":{"address":"counter-n8dh7-head-svc.test-ns-kbx4w.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-11-03T15:17:19.044Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"1ef0946e-4986-42c4-919d-7e4ec9f4c6a6","rayStartCmd":"ray start  --address=counter-n8dh7-head-svc.test-ns-kbx4w.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-11-03T15:17:19.044Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"1ef0946e-4986-42c4-919d-7e4ec9f4c6a6","rayNodeType":"worker","generatedCmd":"ulimit -n ${RAY_START_ULIMIT_OPEN_FILES:-65536}; ray start  --address=counter-n8dh7-head-svc.test-ns-kbx4w.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-11-03T15:17:19.044Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"1ef0946e-4986-42c4-919d-7e4ec9f4c6a6","enabled":true}
{"level":"info","ts":"2026-11-03T15:17:19.122Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"1ef0946e-4986-42c4-919d-7e4ec9f4c6a6","name":"counter-n8dh7-small-group-worker-6tgfv"}
{"level":"info","ts":"2026-11-03T15:17:19.122Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"1ef0946e-4986-42c4-919d-7e4ec9f4c6a6","podName":"counter-n8dh7-head-cshcl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:17:19.122Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"1ef0946e-4986-42c4-919d-7e4ec9f4c6a6","podName":"counter-n8dh7-head-cshcl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:17:19.122Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"1ef0946e-4986-42c4-919d-7e4ec9f4c6a6","name":"counter-n8dh7","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:17:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:17:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:17:19.144Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"d88b561f-9f46-4cd4-a23f-0e5c14b8b46e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:17:19.144Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"d88b561f-9f46-4cd4-a23f-0e5c14b8b46e","name":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:17:19.144Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"d88b561f-9f46-4cd4-a23f-0e5c14b8b46e","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:17:19.144Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"d88b561f-9f46-4cd4-a23f-0e5c14b8b46e"}
{"level":"info","ts":"2026-11-03T15:17:19.144Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"d88b561f-9f46-4cd4-a23f-0e5c14b8b46e","RayCluster":"counter-n8dh7","State":""}
{"level":"info","ts":"2026-11-03T15:17:19.144Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"d88b561f-9f46-4cd4-a23f-0e5c14b8b46e","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:17:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:17:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:17:19.215Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"282f88bc-0a72-4d4d-a55c-ab6b096bf0af"}
{"level":"info","ts":"2026-11-03T15:17:19.215Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"282f88bc-0a72-4d4d-a55c-ab6b096bf0af"}
{"level":"info","ts":"2026-11-03T15:17:19.215Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"282f88bc-0a72-4d4d-a55c-ab6b096bf0af","1 head service found":"counter-n8dh7-head-svc"}
{"level":"info","ts":"2026-11-03T15:17:19.215Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"282f88bc-0a72-4d4d-a55c-ab6b096bf0af","Found 1 head Pod":"counter-n8dh7-head-cshcl","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:17:19.215Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"282f88bc-0a72-4d4d-a55c-ab6b096bf0af","head Pod":"counter-n8dh7-head-cshcl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-n8dh7-head-cshcl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:17:19.215Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"282f88bc-0a72-4d4d-a55c-ab6b096bf0af","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:17:19.215Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"282f88bc-0a72-4d4d-a55c-ab6b096bf0af","worker Pod":"counter-n8dh7-small-group-worker-6tgfv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-n8dh7-small-group-worker-6tgfv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:17:19.215Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"282f88bc-0a72-4d4d-a55c-ab6b096bf0af","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:17:19.215Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"282f88bc-0a72-4d4d-a55c-ab6b096bf0af","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:17:19.215Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"282f88bc-0a72-4d4d-a55c-ab6b096bf0af","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:17:19.215Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"282f88bc-0a72-4d4d-a55c-ab6b096bf0af","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:17:19.215Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"282f88bc-0a72-4d4d-a55c-ab6b096bf0af","podName":"counter-n8dh7-head-cshcl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:17:19.216Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"282f88bc-0a72-4d4d-a55c-ab6b096bf0af","podName":"counter-n8dh7-head-cshcl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:17:19.216Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"4b3f31f9-699c-4f85-b263-bd8efb54e1bb","cluster":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:17:19.216Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"282f88bc-0a72-4d4d-a55c-ab6b096bf0af","name":"counter-n8dh7","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:17:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:17:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:17:19.216Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"98445eae-6a7e-44d0-9121-9d60e45aeeab","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:17:19.216Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"98445eae-6a7e-44d0-9121-9d60e45aeeab","name":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:17:19.216Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"98445eae-6a7e-44d0-9121-9d60e45aeeab","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:17:19.216Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"98445eae-6a7e-44d0-9121-9d60e45aeeab"}
{"level":"info","ts":"2026-11-03T15:17:19.216Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"98445eae-6a7e-44d0-9121-9d60e45aeeab","RayCluster":"counter-n8dh7","State":""}
{"level":"info","ts":"2026-11-03T15:17:19.216Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"98445eae-6a7e-44d0-9121-9d60e45aeeab","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:17:15Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:17:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:17:19.216Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"98445eae-6a7e-44d0-9121-9d60e45aeeab","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:17:19.270Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"282f88bc-0a72-4d4d-a55c-ab6b096bf0af","name":"counter-n8dh7","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-n8dh7\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-kbx4w","name":"counter-n8dh7"}}
{"level":"info","ts":"2026-11-03T15:17:19.270Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"282f88bc-0a72-4d4d-a55c-ab6b096bf0af"}
{"level":"error","ts":"2026-11-03T15:17:19.270Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"282f88bc-0a72-4d4d-a55c-ab6b096bf0af","error":"Operation cannot be fulfilled on rayclusters.ray.io \"counter-n8dh7\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:17:19.270Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"26fa534f-51ea-41df-880d-0b02ba8cc72d"}
{"level":"info","ts":"2026-11-03T15:17:19.270Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"26fa534f-51ea-41df-880d-0b02ba8cc72d"}
{"level":"info","ts":"2026-11-03T15:17:19.270Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"26fa534f-51ea-41df-880d-0b02ba8cc72d","1 head service found":"counter-n8dh7-head-svc"}
{"level":"info","ts":"2026-11-03T15:17:19.270Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"26fa534f-51ea-41df-880d-0b02ba8cc72d","Found 1 head Pod":"counter-n8dh7-head-cshcl","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:17:19.270Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"26fa534f-51ea-41df-880d-0b02ba8cc72d","head Pod":"counter-n8dh7-head-cshcl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-n8dh7-head-cshcl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:17:19.270Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"26fa534f-51ea-41df-880d-0b02ba8cc72d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:17:19.271Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"26fa534f-51ea-41df-880d-0b02ba8cc72d","worker Pod":"counter-n8dh7-small-group-worker-6tgfv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-n8dh7-small-group-worker-6tgfv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:17:19.271Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"26fa534f-51ea-41df-880d-0b02ba8cc72d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:17:19.271Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"26fa534f-51ea-41df-880d-0b02ba8cc72d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:17:19.271Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"26fa534f-51ea-41df-880d-0b02ba8cc72d","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:17:19.271Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"26fa534f-51ea-41df-880d-0b02ba8cc72d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:17:19.271Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"26fa534f-51ea-41df-880d-0b02ba8cc72d","podName":"counter-n8dh7-head-cshcl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:17:19.271Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"26fa534f-51ea-41df-880d-0b02ba8cc72d","podName":"counter-n8dh7-head-cshcl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:17:19.271Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"26fa534f-51ea-41df-880d-0b02ba8cc72d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:17:19.271Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"26fa534f-51ea-41df-880d-0b02ba8cc72d","seconds":300}
{"level":"info","ts":"2026-11-03T15:17:19.323Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"2e405250-ce01-45da-a9f6-18f256725528","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:17:19.323Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"2e405250-ce01-45da-a9f6-18f256725528","name":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:17:19.323Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"2e405250-ce01-45da-a9f6-18f256725528","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:17:19.324Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"2e405250-ce01-45da-a9f6-18f256725528"}
{"level":"info","ts":"2026-11-03T15:17:19.324Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"2e405250-ce01-45da-a9f6-18f256725528","RayCluster":"counter-n8dh7","State":""}
{"level":"info","ts":"2026-11-03T15:17:19.324Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"2e405250-ce01-45da-a9f6-18f256725528","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:17:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:17:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:17:19.739Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"6ef9daab-c441-408f-9d65-4ad91620a6fe"}
{"level":"info","ts":"2026-11-03T15:17:19.739Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"6ef9daab-c441-408f-9d65-4ad91620a6fe"}
{"level":"info","ts":"2026-11-03T15:17:19.740Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"6ef9daab-c441-408f-9d65-4ad91620a6fe","1 head service found":"counter-n8dh7-head-svc"}
{"level":"info","ts":"2026-11-03T15:17:19.740Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"6ef9daab-c441-408f-9d65-4ad91620a6fe","Found 1 head Pod":"counter-n8dh7-head-cshcl","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:17:19.740Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"6ef9daab-c441-408f-9d65-4ad91620a6fe","head Pod":"counter-n8dh7-head-cshcl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-n8dh7-head-cshcl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:17:19.740Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"6ef9daab-c441-408f-9d65-4ad91620a6fe","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:17:19.740Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"6ef9daab-c441-408f-9d65-4ad91620a6fe","worker Pod":"counter-n8dh7-small-group-worker-6tgfv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-n8dh7-small-group-worker-6tgfv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:17:19.740Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"6ef9daab-c441-408f-9d65-4ad91620a6fe","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:17:19.740Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"6ef9daab-c441-408f-9d65-4ad91620a6fe","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:17:19.740Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"6ef9daab-c441-408f-9d65-4ad91620a6fe","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:17:19.740Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"6ef9daab-c441-408f-9d65-4ad91620a6fe","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:17:19.740Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"6ef9daab-c441-408f-9d65-4ad91620a6fe","podName":"counter-n8dh7-head-cshcl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:17:19.740Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"6ef9daab-c441-408f-9d65-4ad91620a6fe","podName":"counter-n8dh7-head-cshcl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:17:19.740Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"6ef9daab-c441-408f-9d65-4ad91620a6fe","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:17:19.740Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"6ef9daab-c441-408f-9d65-4ad91620a6fe","seconds":300}
{"level":"info","ts":"2026-11-03T15:17:19.848Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"66144207-47e7-49d8-b32f-f969bbd24c0a"}
{"level":"info","ts":"2026-11-03T15:17:19.848Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"66144207-47e7-49d8-b32f-f969bbd24c0a"}
{"level":"info","ts":"2026-11-03T15:17:19.848Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"66144207-47e7-49d8-b32f-f969bbd24c0a","1 head service found":"counter-n8dh7-head-svc"}
{"level":"info","ts":"2026-11-03T15:17:19.848Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"66144207-47e7-49d8-b32f-f969bbd24c0a","Found 1 head Pod":"counter-n8dh7-head-cshcl","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:17:19.848Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"66144207-47e7-49d8-b32f-f969bbd24c0a","head Pod":"counter-n8dh7-head-cshcl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-n8dh7-head-cshcl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:17:19.848Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"66144207-47e7-49d8-b32f-f969bbd24c0a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:17:19.848Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"66144207-47e7-49d8-b32f-f969bbd24c0a","worker Pod":"counter-n8dh7-small-group-worker-6tgfv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-n8dh7-small-group-worker-6tgfv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:17:19.848Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"66144207-47e7-49d8-b32f-f969bbd24c0a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:17:19.848Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"66144207-47e7-49d8-b32f-f969bbd24c0a","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:17:19.848Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"66144207-47e7-49d8-b32f-f969bbd24c0a","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:17:19.848Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"66144207-47e7-49d8-b32f-f969bbd24c0a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:17:19.848Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"66144207-47e7-49d8-b32f-f969bbd24c0a","podName":"counter-n8dh7-head-cshcl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:17:19.848Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"66144207-47e7-49d8-b32f-f969bbd24c0a","podName":"counter-n8dh7-head-cshcl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:17:19.848Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"66144207-47e7-49d8-b32f-f969bbd24c0a","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:17:19.848Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"66144207-47e7-49d8-b32f-f969bbd24c0a","seconds":300}
{"level":"info","ts":"2026-11-03T15:17:20.374Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"51e143f2-115d-43ca-8fd9-c3714f9da73c"}
{"level":"info","ts":"2026-11-03T15:17:20.374Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"51e143f2-115d-43ca-8fd9-c3714f9da73c"}
{"level":"info","ts":"2026-11-03T15:17:20.374Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"51e143f2-115d-43ca-8fd9-c3714f9da73c","1 head service found":"counter-n8dh7-head-svc"}
{"level":"info","ts":"2026-11-03T15:17:20.374Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"51e143f2-115d-43ca-8fd9-c3714f9da73c","Found 1 head Pod":"counter-n8dh7-head-cshcl","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:17:20.374Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"51e143f2-115d-43ca-8fd9-c3714f9da73c","head Pod":"counter-n8dh7-head-cshcl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-n8dh7-head-cshcl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:17:20.374Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"51e143f2-115d-43ca-8fd9-c3714f9da73c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:17:20.374Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"51e143f2-115d-43ca-8fd9-c3714f9da73c","worker Pod":"counter-n8dh7-small-group-worker-6tgfv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-n8dh7-small-group-worker-6tgfv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:17:20.374Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"51e143f2-115d-43ca-8fd9-c3714f9da73c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:17:20.374Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"51e143f2-115d-43ca-8fd9-c3714f9da73c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:17:20.374Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"51e143f2-115d-43ca-8fd9-c3714f9da73c","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:17:20.374Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"51e143f2-115d-43ca-8fd9-c3714f9da73c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:17:20.375Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"51e143f2-115d-43ca-8fd9-c3714f9da73c","podName":"counter-n8dh7-head-cshcl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:17:20.375Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"51e143f2-115d-43ca-8fd9-c3714f9da73c","podName":"counter-n8dh7-head-cshcl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:17:20.375Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"51e143f2-115d-43ca-8fd9-c3714f9da73c","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:17:20.375Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"51e143f2-115d-43ca-8fd9-c3714f9da73c","seconds":300}
{"level":"info","ts":"2026-11-03T15:17:20.450Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"fac94c7f-9f1a-40ce-8a8f-b4778efa677b"}
{"level":"info","ts":"2026-11-03T15:17:20.450Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"fac94c7f-9f1a-40ce-8a8f-b4778efa677b"}
{"level":"info","ts":"2026-11-03T15:17:20.451Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"fac94c7f-9f1a-40ce-8a8f-b4778efa677b","1 head service found":"counter-n8dh7-head-svc"}
{"level":"info","ts":"2026-11-03T15:17:20.451Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"fac94c7f-9f1a-40ce-8a8f-b4778efa677b","Found 1 head Pod":"counter-n8dh7-head-cshcl","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:17:20.451Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"fac94c7f-9f1a-40ce-8a8f-b4778efa677b","head Pod":"counter-n8dh7-head-cshcl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-n8dh7-head-cshcl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:17:20.451Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"fac94c7f-9f1a-40ce-8a8f-b4778efa677b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:17:20.451Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"fac94c7f-9f1a-40ce-8a8f-b4778efa677b","worker Pod":"counter-n8dh7-small-group-worker-6tgfv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-n8dh7-small-group-worker-6tgfv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:17:20.451Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"fac94c7f-9f1a-40ce-8a8f-b4778efa677b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:17:20.451Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"fac94c7f-9f1a-40ce-8a8f-b4778efa677b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:17:20.451Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"fac94c7f-9f1a-40ce-8a8f-b4778efa677b","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:17:20.451Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"fac94c7f-9f1a-40ce-8a8f-b4778efa677b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:17:20.451Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"fac94c7f-9f1a-40ce-8a8f-b4778efa677b","podName":"counter-n8dh7-head-cshcl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:17:20.451Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"fac94c7f-9f1a-40ce-8a8f-b4778efa677b","podName":"counter-n8dh7-head-cshcl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:17:20.451Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"fac94c7f-9f1a-40ce-8a8f-b4778efa677b","name":"counter-n8dh7","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:17:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:17:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:17:20.512Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"355b649b-a4d0-40e0-8045-3876aaacad40","cluster":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:17:20.512Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"0ee62ffd-648c-4d08-821a-3d61dd46359f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:17:20.512Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"0ee62ffd-648c-4d08-821a-3d61dd46359f","name":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:17:20.512Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"0ee62ffd-648c-4d08-821a-3d61dd46359f","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:17:20.512Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"0ee62ffd-648c-4d08-821a-3d61dd46359f"}
{"level":"info","ts":"2026-11-03T15:17:20.512Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"0ee62ffd-648c-4d08-821a-3d61dd46359f","RayCluster":"counter-n8dh7","State":""}
{"level":"info","ts":"2026-11-03T15:17:20.512Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"0ee62ffd-648c-4d08-821a-3d61dd46359f","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:17:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:17:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:17:20.512Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"0ee62ffd-648c-4d08-821a-3d61dd46359f","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:17:20.573Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"bc673792-87cb-4b09-b015-f519239531d6","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:17:20.573Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"bc673792-87cb-4b09-b015-f519239531d6","name":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:17:20.573Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"bc673792-87cb-4b09-b015-f519239531d6","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:17:20.573Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"bc673792-87cb-4b09-b015-f519239531d6"}
{"level":"info","ts":"2026-11-03T15:17:20.573Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"bc673792-87cb-4b09-b015-f519239531d6","RayCluster":"counter-n8dh7","State":""}
{"level":"info","ts":"2026-11-03T15:17:20.573Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"bc673792-87cb-4b09-b015-f519239531d6","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:17:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:17:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:17:20.836Z","logger":"controllers.RayClusterMTLS","msg":"Updating head certificate","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"355b649b-a4d0-40e0-8045-3876aaacad40","certificate":"ray-head-cert-counter-n8dh7","ipAddresses":["10.132.0.38","127.0.0.1"]}
{"level":"info","ts":"2026-11-03T15:17:21.093Z","logger":"controllers.RayClusterMTLS","msg":"Updating worker certificate","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"355b649b-a4d0-40e0-8045-3876aaacad40","certificate":"ray-worker-cert-counter-n8dh7","ipAddresses":["10.134.0.37","127.0.0.1"]}
{"level":"info","ts":"2026-11-03T15:17:21.347Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"355b649b-a4d0-40e0-8045-3876aaacad40","certificate":"ray-head-cert-counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:17:21.347Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"355b649b-a4d0-40e0-8045-3876aaacad40"}
{"level":"info","ts":"2026-11-03T15:17:22.513Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"ba2e8443-c902-4b5a-9c0c-7e9f7603a5e7"}
{"level":"info","ts":"2026-11-03T15:17:22.513Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"ba2e8443-c902-4b5a-9c0c-7e9f7603a5e7"}
{"level":"info","ts":"2026-11-03T15:17:22.513Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"ba2e8443-c902-4b5a-9c0c-7e9f7603a5e7","1 head service found":"counter-n8dh7-head-svc"}
{"level":"info","ts":"2026-11-03T15:17:22.513Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"ba2e8443-c902-4b5a-9c0c-7e9f7603a5e7","Found 1 head Pod":"counter-n8dh7-head-cshcl","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:17:22.513Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"ba2e8443-c902-4b5a-9c0c-7e9f7603a5e7","head Pod":"counter-n8dh7-head-cshcl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-n8dh7-head-cshcl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:17:22.513Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"ba2e8443-c902-4b5a-9c0c-7e9f7603a5e7","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:17:22.513Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"ba2e8443-c902-4b5a-9c0c-7e9f7603a5e7","worker Pod":"counter-n8dh7-small-group-worker-6tgfv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-n8dh7-small-group-worker-6tgfv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:17:22.513Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"ba2e8443-c902-4b5a-9c0c-7e9f7603a5e7","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:17:22.513Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"ba2e8443-c902-4b5a-9c0c-7e9f7603a5e7","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:17:22.513Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"ba2e8443-c902-4b5a-9c0c-7e9f7603a5e7","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:17:22.513Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"ba2e8443-c902-4b5a-9c0c-7e9f7603a5e7","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:17:22.513Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"ba2e8443-c902-4b5a-9c0c-7e9f7603a5e7","podName":"counter-n8dh7-head-cshcl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:17:22.513Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"ba2e8443-c902-4b5a-9c0c-7e9f7603a5e7","podName":"counter-n8dh7-head-cshcl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:17:22.513Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"ba2e8443-c902-4b5a-9c0c-7e9f7603a5e7","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:17:22.513Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"ba2e8443-c902-4b5a-9c0c-7e9f7603a5e7","seconds":300}
{"level":"info","ts":"2026-11-03T15:17:23.574Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"359e85af-1673-4569-aaaf-117dba826602","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:17:23.574Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"359e85af-1673-4569-aaaf-117dba826602","name":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:17:23.574Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"359e85af-1673-4569-aaaf-117dba826602","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:17:23.574Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"359e85af-1673-4569-aaaf-117dba826602"}
{"level":"info","ts":"2026-11-03T15:17:23.574Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"359e85af-1673-4569-aaaf-117dba826602","RayCluster":"counter-n8dh7","State":""}
{"level":"info","ts":"2026-11-03T15:17:23.574Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"359e85af-1673-4569-aaaf-117dba826602","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:17:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:17:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:17:26.575Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"eab3bba1-9132-44c7-ab7c-a7de07fb90d2","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:17:26.575Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"eab3bba1-9132-44c7-ab7c-a7de07fb90d2","name":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:17:26.575Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"eab3bba1-9132-44c7-ab7c-a7de07fb90d2","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:17:26.575Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"eab3bba1-9132-44c7-ab7c-a7de07fb90d2"}
{"level":"info","ts":"2026-11-03T15:17:26.575Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"eab3bba1-9132-44c7-ab7c-a7de07fb90d2","RayCluster":"counter-n8dh7","State":""}
{"level":"info","ts":"2026-11-03T15:17:26.575Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"eab3bba1-9132-44c7-ab7c-a7de07fb90d2","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:17:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:17:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:17:29.575Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"2347d030-7604-48b8-8793-1dff3d5d5755","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:17:29.575Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"2347d030-7604-48b8-8793-1dff3d5d5755","name":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:17:29.576Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"2347d030-7604-48b8-8793-1dff3d5d5755","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:17:29.576Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"2347d030-7604-48b8-8793-1dff3d5d5755"}
{"level":"info","ts":"2026-11-03T15:17:29.576Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"2347d030-7604-48b8-8793-1dff3d5d5755","RayCluster":"counter-n8dh7","State":""}
{"level":"info","ts":"2026-11-03T15:17:29.576Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"2347d030-7604-48b8-8793-1dff3d5d5755","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:17:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:17:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:17:32.577Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"6e312cc8-405b-4e3e-982d-c8019ca0731c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:17:32.577Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"6e312cc8-405b-4e3e-982d-c8019ca0731c","name":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:17:32.577Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"6e312cc8-405b-4e3e-982d-c8019ca0731c","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:17:32.577Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"6e312cc8-405b-4e3e-982d-c8019ca0731c"}
{"level":"info","ts":"2026-11-03T15:17:32.577Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"6e312cc8-405b-4e3e-982d-c8019ca0731c","RayCluster":"counter-n8dh7","State":""}
{"level":"info","ts":"2026-11-03T15:17:32.577Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"6e312cc8-405b-4e3e-982d-c8019ca0731c","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:17:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:17:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:17:35.578Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"d539d674-0dbe-4735-b6ab-282cef8a842a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:17:35.578Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"d539d674-0dbe-4735-b6ab-282cef8a842a","name":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:17:35.578Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"d539d674-0dbe-4735-b6ab-282cef8a842a","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:17:35.579Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"d539d674-0dbe-4735-b6ab-282cef8a842a"}
{"level":"info","ts":"2026-11-03T15:17:35.579Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"d539d674-0dbe-4735-b6ab-282cef8a842a","RayCluster":"counter-n8dh7","State":""}
{"level":"info","ts":"2026-11-03T15:17:35.579Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"d539d674-0dbe-4735-b6ab-282cef8a842a","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:17:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:17:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:17:38.580Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"4a7864df-c6f0-42f2-a24d-900647c73fa1","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:17:38.580Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"4a7864df-c6f0-42f2-a24d-900647c73fa1","name":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:17:38.580Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"4a7864df-c6f0-42f2-a24d-900647c73fa1","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:17:38.580Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"4a7864df-c6f0-42f2-a24d-900647c73fa1"}
{"level":"info","ts":"2026-11-03T15:17:38.580Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"4a7864df-c6f0-42f2-a24d-900647c73fa1","RayCluster":"counter-n8dh7","State":""}
{"level":"info","ts":"2026-11-03T15:17:38.580Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"4a7864df-c6f0-42f2-a24d-900647c73fa1","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:17:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:17:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:17:41.581Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"5e9dd830-d1b3-4e17-9e41-e352e4632433","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:17:41.581Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"5e9dd830-d1b3-4e17-9e41-e352e4632433","name":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:17:41.581Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"5e9dd830-d1b3-4e17-9e41-e352e4632433","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:17:41.581Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"5e9dd830-d1b3-4e17-9e41-e352e4632433"}
{"level":"info","ts":"2026-11-03T15:17:41.581Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"5e9dd830-d1b3-4e17-9e41-e352e4632433","RayCluster":"counter-n8dh7","State":""}
{"level":"info","ts":"2026-11-03T15:17:41.581Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"5e9dd830-d1b3-4e17-9e41-e352e4632433","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:17:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:17:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:17:44.582Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"7c61a03f-3fb6-4b69-89c5-c7bee97f52b0","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:17:44.582Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"7c61a03f-3fb6-4b69-89c5-c7bee97f52b0","name":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:17:44.582Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"7c61a03f-3fb6-4b69-89c5-c7bee97f52b0","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:17:44.582Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"7c61a03f-3fb6-4b69-89c5-c7bee97f52b0"}
{"level":"info","ts":"2026-11-03T15:17:44.582Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"7c61a03f-3fb6-4b69-89c5-c7bee97f52b0","RayCluster":"counter-n8dh7","State":""}
{"level":"info","ts":"2026-11-03T15:17:44.582Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"7c61a03f-3fb6-4b69-89c5-c7bee97f52b0","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:17:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:17:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:17:47.583Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"5eb08873-16a9-470e-b78c-3d532ca14441","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:17:47.583Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"5eb08873-16a9-470e-b78c-3d532ca14441","name":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:17:47.583Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"5eb08873-16a9-470e-b78c-3d532ca14441","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:17:47.583Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"5eb08873-16a9-470e-b78c-3d532ca14441"}
{"level":"info","ts":"2026-11-03T15:17:47.583Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"5eb08873-16a9-470e-b78c-3d532ca14441","RayCluster":"counter-n8dh7","State":""}
{"level":"info","ts":"2026-11-03T15:17:47.583Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"5eb08873-16a9-470e-b78c-3d532ca14441","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:17:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:17:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:17:50.584Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"9cbe87a6-072b-4a83-b7bd-7440b4d50557","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:17:50.584Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"9cbe87a6-072b-4a83-b7bd-7440b4d50557","name":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:17:50.584Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"9cbe87a6-072b-4a83-b7bd-7440b4d50557","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:17:50.584Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"9cbe87a6-072b-4a83-b7bd-7440b4d50557"}
{"level":"info","ts":"2026-11-03T15:17:50.584Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"9cbe87a6-072b-4a83-b7bd-7440b4d50557","RayCluster":"counter-n8dh7","State":""}
{"level":"info","ts":"2026-11-03T15:17:50.584Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"9cbe87a6-072b-4a83-b7bd-7440b4d50557","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:17:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:17:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:17:51.347Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"72511ab3-b558-4070-a481-300b9b774582","cluster":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:17:53.585Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"29eae689-3154-47d2-b514-9af8f6387749","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:17:53.585Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"29eae689-3154-47d2-b514-9af8f6387749","name":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:17:53.585Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"29eae689-3154-47d2-b514-9af8f6387749","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:17:53.585Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"29eae689-3154-47d2-b514-9af8f6387749"}
{"level":"info","ts":"2026-11-03T15:17:53.585Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"29eae689-3154-47d2-b514-9af8f6387749","RayCluster":"counter-n8dh7","State":""}
{"level":"info","ts":"2026-11-03T15:17:53.585Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"29eae689-3154-47d2-b514-9af8f6387749","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:17:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:17:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:17:56.586Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"3f107267-1b6f-4206-9ca4-33f8c8f77110","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:17:56.586Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"3f107267-1b6f-4206-9ca4-33f8c8f77110","name":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:17:56.586Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"3f107267-1b6f-4206-9ca4-33f8c8f77110","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:17:56.586Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"3f107267-1b6f-4206-9ca4-33f8c8f77110"}
{"level":"info","ts":"2026-11-03T15:17:56.586Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"3f107267-1b6f-4206-9ca4-33f8c8f77110","RayCluster":"counter-n8dh7","State":""}
{"level":"info","ts":"2026-11-03T15:17:56.586Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"3f107267-1b6f-4206-9ca4-33f8c8f77110","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:17:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:17:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:17:59.587Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"55532af2-bbfd-4bdf-aed2-fe8a525b1e8e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:17:59.587Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"55532af2-bbfd-4bdf-aed2-fe8a525b1e8e","name":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:17:59.587Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"55532af2-bbfd-4bdf-aed2-fe8a525b1e8e","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:17:59.588Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"55532af2-bbfd-4bdf-aed2-fe8a525b1e8e"}
{"level":"info","ts":"2026-11-03T15:17:59.588Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"55532af2-bbfd-4bdf-aed2-fe8a525b1e8e","RayCluster":"counter-n8dh7","State":""}
{"level":"info","ts":"2026-11-03T15:17:59.588Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"55532af2-bbfd-4bdf-aed2-fe8a525b1e8e","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:17:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:17:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:18:02.589Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"7e6c0bd0-60f9-4992-8573-770414085d37","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:18:02.589Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"7e6c0bd0-60f9-4992-8573-770414085d37","name":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:18:02.589Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"7e6c0bd0-60f9-4992-8573-770414085d37","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:18:02.589Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"7e6c0bd0-60f9-4992-8573-770414085d37"}
{"level":"info","ts":"2026-11-03T15:18:02.589Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"7e6c0bd0-60f9-4992-8573-770414085d37","RayCluster":"counter-n8dh7","State":""}
{"level":"info","ts":"2026-11-03T15:18:02.589Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"7e6c0bd0-60f9-4992-8573-770414085d37","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:17:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:17:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:18:05.590Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"873f85b9-c09e-4947-a880-7c8e0def0831","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:18:05.590Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"873f85b9-c09e-4947-a880-7c8e0def0831","name":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:18:05.590Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"873f85b9-c09e-4947-a880-7c8e0def0831","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:18:05.590Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"873f85b9-c09e-4947-a880-7c8e0def0831"}
{"level":"info","ts":"2026-11-03T15:18:05.590Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"873f85b9-c09e-4947-a880-7c8e0def0831","RayCluster":"counter-n8dh7","State":""}
{"level":"info","ts":"2026-11-03T15:18:05.590Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"873f85b9-c09e-4947-a880-7c8e0def0831","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:17:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:17:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:18:08.591Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"48552a91-611c-403f-9930-e199be006fa3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:18:08.591Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"48552a91-611c-403f-9930-e199be006fa3","name":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:18:08.591Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"48552a91-611c-403f-9930-e199be006fa3","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:18:08.591Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"48552a91-611c-403f-9930-e199be006fa3"}
{"level":"info","ts":"2026-11-03T15:18:08.591Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"48552a91-611c-403f-9930-e199be006fa3","RayCluster":"counter-n8dh7","State":""}
{"level":"info","ts":"2026-11-03T15:18:08.591Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"48552a91-611c-403f-9930-e199be006fa3","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:17:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:17:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:18:11.593Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"15b742b1-6615-414a-9139-e7cc75349e79","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:18:11.593Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"15b742b1-6615-414a-9139-e7cc75349e79","name":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:18:11.593Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"15b742b1-6615-414a-9139-e7cc75349e79","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:18:11.593Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"15b742b1-6615-414a-9139-e7cc75349e79"}
{"level":"info","ts":"2026-11-03T15:18:11.593Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"15b742b1-6615-414a-9139-e7cc75349e79","RayCluster":"counter-n8dh7","State":""}
{"level":"info","ts":"2026-11-03T15:18:11.593Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"15b742b1-6615-414a-9139-e7cc75349e79","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:17:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:17:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:18:14.593Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"97e8313b-4391-4c2a-9eb7-f26efd4ad677","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:18:14.593Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"97e8313b-4391-4c2a-9eb7-f26efd4ad677","name":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:18:14.593Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"97e8313b-4391-4c2a-9eb7-f26efd4ad677","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:18:14.594Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"97e8313b-4391-4c2a-9eb7-f26efd4ad677"}
{"level":"info","ts":"2026-11-03T15:18:14.594Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"97e8313b-4391-4c2a-9eb7-f26efd4ad677","RayCluster":"counter-n8dh7","State":""}
{"level":"info","ts":"2026-11-03T15:18:14.594Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"97e8313b-4391-4c2a-9eb7-f26efd4ad677","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:17:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:17:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:18:17.594Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"1df9c285-1696-4f7e-8a12-08b328bc7113","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:18:17.594Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"1df9c285-1696-4f7e-8a12-08b328bc7113","name":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:18:17.595Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"1df9c285-1696-4f7e-8a12-08b328bc7113","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:18:17.595Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"1df9c285-1696-4f7e-8a12-08b328bc7113"}
{"level":"info","ts":"2026-11-03T15:18:17.595Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"1df9c285-1696-4f7e-8a12-08b328bc7113","RayCluster":"counter-n8dh7","State":""}
{"level":"info","ts":"2026-11-03T15:18:17.595Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"1df9c285-1696-4f7e-8a12-08b328bc7113","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:17:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:17:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:18:20.596Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"a1ff3a32-b285-4cd4-8ad6-9815b6acd7c1","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:18:20.596Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"a1ff3a32-b285-4cd4-8ad6-9815b6acd7c1","name":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:18:20.596Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"a1ff3a32-b285-4cd4-8ad6-9815b6acd7c1","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:18:20.596Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"a1ff3a32-b285-4cd4-8ad6-9815b6acd7c1"}
{"level":"info","ts":"2026-11-03T15:18:20.596Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"a1ff3a32-b285-4cd4-8ad6-9815b6acd7c1","RayCluster":"counter-n8dh7","State":""}
{"level":"info","ts":"2026-11-03T15:18:20.596Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"a1ff3a32-b285-4cd4-8ad6-9815b6acd7c1","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:17:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:17:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:18:23.597Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"0ffd2373-ef6c-41b2-af2c-7b31275a7287","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:18:23.597Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"0ffd2373-ef6c-41b2-af2c-7b31275a7287","name":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:18:23.597Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"0ffd2373-ef6c-41b2-af2c-7b31275a7287","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:18:23.597Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"0ffd2373-ef6c-41b2-af2c-7b31275a7287"}
{"level":"info","ts":"2026-11-03T15:18:23.597Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"0ffd2373-ef6c-41b2-af2c-7b31275a7287","RayCluster":"counter-n8dh7","State":""}
{"level":"info","ts":"2026-11-03T15:18:23.597Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"0ffd2373-ef6c-41b2-af2c-7b31275a7287","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:17:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:17:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:18:26.598Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"3ffc827e-0fe5-4d98-b655-61e7f06b0532","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:18:26.598Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"3ffc827e-0fe5-4d98-b655-61e7f06b0532","name":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:18:26.598Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"3ffc827e-0fe5-4d98-b655-61e7f06b0532","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:18:26.598Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"3ffc827e-0fe5-4d98-b655-61e7f06b0532"}
{"level":"info","ts":"2026-11-03T15:18:26.598Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"3ffc827e-0fe5-4d98-b655-61e7f06b0532","RayCluster":"counter-n8dh7","State":""}
{"level":"info","ts":"2026-11-03T15:18:26.598Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"3ffc827e-0fe5-4d98-b655-61e7f06b0532","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:17:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:17:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:18:29.599Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"6595357d-4241-4669-b4ba-898d37ee34a2","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:18:29.599Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"6595357d-4241-4669-b4ba-898d37ee34a2","name":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:18:29.600Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"6595357d-4241-4669-b4ba-898d37ee34a2","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:18:29.600Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"6595357d-4241-4669-b4ba-898d37ee34a2"}
{"level":"info","ts":"2026-11-03T15:18:29.600Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"6595357d-4241-4669-b4ba-898d37ee34a2","RayCluster":"counter-n8dh7","State":""}
{"level":"info","ts":"2026-11-03T15:18:29.600Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"6595357d-4241-4669-b4ba-898d37ee34a2","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:17:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:17:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:18:32.600Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"a77d0730-8c92-4847-9060-2719f76a5f16","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:18:32.600Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"a77d0730-8c92-4847-9060-2719f76a5f16","name":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:18:32.600Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"a77d0730-8c92-4847-9060-2719f76a5f16","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:18:32.601Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"a77d0730-8c92-4847-9060-2719f76a5f16"}
{"level":"info","ts":"2026-11-03T15:18:32.601Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"a77d0730-8c92-4847-9060-2719f76a5f16","RayCluster":"counter-n8dh7","State":""}
{"level":"info","ts":"2026-11-03T15:18:32.601Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"a77d0730-8c92-4847-9060-2719f76a5f16","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:17:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:17:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:18:35.602Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"4eb82963-3151-4312-8344-7a4446160d4f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:18:35.602Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"4eb82963-3151-4312-8344-7a4446160d4f","name":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:18:35.602Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"4eb82963-3151-4312-8344-7a4446160d4f","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:18:35.602Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"4eb82963-3151-4312-8344-7a4446160d4f"}
{"level":"info","ts":"2026-11-03T15:18:35.602Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"4eb82963-3151-4312-8344-7a4446160d4f","RayCluster":"counter-n8dh7","State":""}
{"level":"info","ts":"2026-11-03T15:18:35.602Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"4eb82963-3151-4312-8344-7a4446160d4f","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:17:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:17:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:18:36.588Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"2a225dd7-1f7b-4271-a8e9-86af5b987fb5"}
{"level":"info","ts":"2026-11-03T15:18:36.588Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"2a225dd7-1f7b-4271-a8e9-86af5b987fb5"}
{"level":"info","ts":"2026-11-03T15:18:36.588Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"2a225dd7-1f7b-4271-a8e9-86af5b987fb5","1 head service found":"counter-n8dh7-head-svc"}
{"level":"info","ts":"2026-11-03T15:18:36.588Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"2a225dd7-1f7b-4271-a8e9-86af5b987fb5","Found 1 head Pod":"counter-n8dh7-head-cshcl","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:18:36.588Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"2a225dd7-1f7b-4271-a8e9-86af5b987fb5","head Pod":"counter-n8dh7-head-cshcl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-n8dh7-head-cshcl. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:18:36.588Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"2a225dd7-1f7b-4271-a8e9-86af5b987fb5","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:18:36.588Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"2a225dd7-1f7b-4271-a8e9-86af5b987fb5","worker Pod":"counter-n8dh7-small-group-worker-6tgfv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-n8dh7-small-group-worker-6tgfv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:18:36.588Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"2a225dd7-1f7b-4271-a8e9-86af5b987fb5","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:18:36.588Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"2a225dd7-1f7b-4271-a8e9-86af5b987fb5","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:18:36.588Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"2a225dd7-1f7b-4271-a8e9-86af5b987fb5","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:18:36.588Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"2a225dd7-1f7b-4271-a8e9-86af5b987fb5","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:18:36.588Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"2a225dd7-1f7b-4271-a8e9-86af5b987fb5","podName":"counter-n8dh7-head-cshcl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:18:36.588Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"2a225dd7-1f7b-4271-a8e9-86af5b987fb5","podName":"counter-n8dh7-head-cshcl","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:18:36.588Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"2a225dd7-1f7b-4271-a8e9-86af5b987fb5","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:18:36.588Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"2a225dd7-1f7b-4271-a8e9-86af5b987fb5","seconds":300}
{"level":"info","ts":"2026-11-03T15:18:36.777Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"392ce0c8-1e09-483c-98a6-53216496b11f"}
{"level":"info","ts":"2026-11-03T15:18:36.777Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"392ce0c8-1e09-483c-98a6-53216496b11f"}
{"level":"info","ts":"2026-11-03T15:18:36.778Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"392ce0c8-1e09-483c-98a6-53216496b11f","1 head service found":"counter-n8dh7-head-svc"}
{"level":"info","ts":"2026-11-03T15:18:36.778Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"392ce0c8-1e09-483c-98a6-53216496b11f","Found 1 head Pod":"counter-n8dh7-head-cshcl","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:18:36.778Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"392ce0c8-1e09-483c-98a6-53216496b11f","head Pod":"counter-n8dh7-head-cshcl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-n8dh7-head-cshcl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:18:36.778Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"392ce0c8-1e09-483c-98a6-53216496b11f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:18:36.778Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"392ce0c8-1e09-483c-98a6-53216496b11f","worker Pod":"counter-n8dh7-small-group-worker-6tgfv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-n8dh7-small-group-worker-6tgfv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:18:36.778Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"392ce0c8-1e09-483c-98a6-53216496b11f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:18:36.778Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"392ce0c8-1e09-483c-98a6-53216496b11f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:18:36.778Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"392ce0c8-1e09-483c-98a6-53216496b11f","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:18:36.778Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"392ce0c8-1e09-483c-98a6-53216496b11f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:18:36.778Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"392ce0c8-1e09-483c-98a6-53216496b11f","podName":"counter-n8dh7-head-cshcl","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:17:18Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:18:36.778Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"392ce0c8-1e09-483c-98a6-53216496b11f","podName":"counter-n8dh7-head-cshcl","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:17:18Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:18:36.778Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"392ce0c8-1e09-483c-98a6-53216496b11f","name":"counter-n8dh7","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:17:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:18:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:18:36.844Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"912d6e1d-2b9a-4d32-afa3-15fbda6ff869","cluster":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:18:36.844Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"16e00d1d-8a9c-47a4-8a66-34d5665167a8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:18:36.844Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"16e00d1d-8a9c-47a4-8a66-34d5665167a8","name":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:18:36.844Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"16e00d1d-8a9c-47a4-8a66-34d5665167a8","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:18:36.844Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"16e00d1d-8a9c-47a4-8a66-34d5665167a8"}
{"level":"info","ts":"2026-11-03T15:18:36.844Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"16e00d1d-8a9c-47a4-8a66-34d5665167a8","RayCluster":"counter-n8dh7","State":""}
{"level":"info","ts":"2026-11-03T15:18:36.844Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"16e00d1d-8a9c-47a4-8a66-34d5665167a8","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:17:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:18:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:18:36.844Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"16e00d1d-8a9c-47a4-8a66-34d5665167a8","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:18:36.976Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"380d830a-b14a-4c2d-9081-af9375cdd08a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:18:36.976Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"380d830a-b14a-4c2d-9081-af9375cdd08a","name":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:18:36.976Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"380d830a-b14a-4c2d-9081-af9375cdd08a","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:18:36.976Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"380d830a-b14a-4c2d-9081-af9375cdd08a"}
{"level":"info","ts":"2026-11-03T15:18:36.976Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"380d830a-b14a-4c2d-9081-af9375cdd08a","RayCluster":"counter-n8dh7","State":""}
{"level":"info","ts":"2026-11-03T15:18:36.976Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"380d830a-b14a-4c2d-9081-af9375cdd08a","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:18:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:18:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:18:38.890Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"92166ef8-029d-4f20-bda1-54e7a544de22"}
{"level":"info","ts":"2026-11-03T15:18:38.890Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"92166ef8-029d-4f20-bda1-54e7a544de22"}
{"level":"info","ts":"2026-11-03T15:18:38.890Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"92166ef8-029d-4f20-bda1-54e7a544de22","1 head service found":"counter-n8dh7-head-svc"}
{"level":"info","ts":"2026-11-03T15:18:38.890Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"92166ef8-029d-4f20-bda1-54e7a544de22","Found 1 head Pod":"counter-n8dh7-head-cshcl","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:18:38.890Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"92166ef8-029d-4f20-bda1-54e7a544de22","head Pod":"counter-n8dh7-head-cshcl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-n8dh7-head-cshcl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:18:38.890Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"92166ef8-029d-4f20-bda1-54e7a544de22","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:18:38.891Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"92166ef8-029d-4f20-bda1-54e7a544de22","worker Pod":"counter-n8dh7-small-group-worker-6tgfv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-n8dh7-small-group-worker-6tgfv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:18:38.891Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"92166ef8-029d-4f20-bda1-54e7a544de22","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:18:38.891Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"92166ef8-029d-4f20-bda1-54e7a544de22","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:18:38.891Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"92166ef8-029d-4f20-bda1-54e7a544de22","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:18:38.891Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"92166ef8-029d-4f20-bda1-54e7a544de22","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:18:38.891Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"92166ef8-029d-4f20-bda1-54e7a544de22","podName":"counter-n8dh7-small-group-worker-6tgfv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:18:38.891Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"92166ef8-029d-4f20-bda1-54e7a544de22","podName":"counter-n8dh7-small-group-worker-6tgfv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:18:38.891Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"92166ef8-029d-4f20-bda1-54e7a544de22","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:18:38.891Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"92166ef8-029d-4f20-bda1-54e7a544de22","seconds":300}
{"level":"info","ts":"2026-11-03T15:18:39.977Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"c877e00c-475c-4d9d-97ce-942c5df1ded7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:18:39.977Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"c877e00c-475c-4d9d-97ce-942c5df1ded7","name":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:18:39.977Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"c877e00c-475c-4d9d-97ce-942c5df1ded7","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:18:39.977Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"c877e00c-475c-4d9d-97ce-942c5df1ded7"}
{"level":"info","ts":"2026-11-03T15:18:39.977Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"c877e00c-475c-4d9d-97ce-942c5df1ded7","RayCluster":"counter-n8dh7","State":""}
{"level":"info","ts":"2026-11-03T15:18:39.977Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"c877e00c-475c-4d9d-97ce-942c5df1ded7","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:18:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:18:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:18:42.978Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"69968bcc-1e25-45ac-b49f-d5c208d69786","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:18:42.978Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"69968bcc-1e25-45ac-b49f-d5c208d69786","name":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:18:42.978Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"69968bcc-1e25-45ac-b49f-d5c208d69786","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:18:42.978Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"69968bcc-1e25-45ac-b49f-d5c208d69786"}
{"level":"info","ts":"2026-11-03T15:18:42.978Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"69968bcc-1e25-45ac-b49f-d5c208d69786","RayCluster":"counter-n8dh7","State":""}
{"level":"info","ts":"2026-11-03T15:18:42.978Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"69968bcc-1e25-45ac-b49f-d5c208d69786","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:18:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:18:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:18:45.979Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"829c0da1-015f-4bf4-9533-d68fbd987160","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:18:45.979Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"829c0da1-015f-4bf4-9533-d68fbd987160","name":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:18:45.979Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"829c0da1-015f-4bf4-9533-d68fbd987160","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:18:45.979Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"829c0da1-015f-4bf4-9533-d68fbd987160"}
{"level":"info","ts":"2026-11-03T15:18:45.979Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"829c0da1-015f-4bf4-9533-d68fbd987160","RayCluster":"counter-n8dh7","State":""}
{"level":"info","ts":"2026-11-03T15:18:45.979Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"829c0da1-015f-4bf4-9533-d68fbd987160","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:18:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:18:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:18:47.847Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"0d34c000-a428-43a8-a7c6-da5f4c940025"}
{"level":"info","ts":"2026-11-03T15:18:47.847Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"0d34c000-a428-43a8-a7c6-da5f4c940025"}
{"level":"info","ts":"2026-11-03T15:18:47.847Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"0d34c000-a428-43a8-a7c6-da5f4c940025","1 head service found":"counter-n8dh7-head-svc"}
{"level":"info","ts":"2026-11-03T15:18:47.847Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"0d34c000-a428-43a8-a7c6-da5f4c940025","Found 1 head Pod":"counter-n8dh7-head-cshcl","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:18:47.847Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"0d34c000-a428-43a8-a7c6-da5f4c940025","head Pod":"counter-n8dh7-head-cshcl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-n8dh7-head-cshcl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:18:47.847Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"0d34c000-a428-43a8-a7c6-da5f4c940025","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:18:47.847Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"0d34c000-a428-43a8-a7c6-da5f4c940025","worker Pod":"counter-n8dh7-small-group-worker-6tgfv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-n8dh7-small-group-worker-6tgfv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:18:47.847Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"0d34c000-a428-43a8-a7c6-da5f4c940025","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:18:47.847Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"0d34c000-a428-43a8-a7c6-da5f4c940025","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:18:47.847Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"0d34c000-a428-43a8-a7c6-da5f4c940025","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:18:47.847Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"0d34c000-a428-43a8-a7c6-da5f4c940025","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:18:47.847Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"0d34c000-a428-43a8-a7c6-da5f4c940025","podName":"counter-n8dh7-small-group-worker-6tgfv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:18:47.847Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"0d34c000-a428-43a8-a7c6-da5f4c940025","podName":"counter-n8dh7-small-group-worker-6tgfv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:18:47.847Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"0d34c000-a428-43a8-a7c6-da5f4c940025","name":"counter-n8dh7","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:18:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:18:47Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:18:47Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:18:47.910Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"48cab9f1-dc33-4c91-9e50-2674f7e777f7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:18:47.910Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"48cab9f1-dc33-4c91-9e50-2674f7e777f7","name":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:18:47.910Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"48cab9f1-dc33-4c91-9e50-2674f7e777f7","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:18:47.910Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"be5a7ef0-ec4c-467d-9599-ba47470ef325","cluster":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:18:47.910Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"48cab9f1-dc33-4c91-9e50-2674f7e777f7"}
{"level":"info","ts":"2026-11-03T15:18:47.910Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"48cab9f1-dc33-4c91-9e50-2674f7e777f7","RayCluster":"counter-n8dh7","State":""}
{"level":"info","ts":"2026-11-03T15:18:47.910Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"48cab9f1-dc33-4c91-9e50-2674f7e777f7","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:18:36Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:18:47Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:18:47Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:18:47.911Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"48cab9f1-dc33-4c91-9e50-2674f7e777f7","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:18:48.046Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"fb29de63-38ba-469a-bb0c-c61d5147092d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:18:48.046Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"fb29de63-38ba-469a-bb0c-c61d5147092d","name":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:18:48.046Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"fb29de63-38ba-469a-bb0c-c61d5147092d","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:18:48.046Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"fb29de63-38ba-469a-bb0c-c61d5147092d"}
{"level":"info","ts":"2026-11-03T15:18:48.046Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"fb29de63-38ba-469a-bb0c-c61d5147092d","RayCluster":"counter-n8dh7","State":""}
{"level":"info","ts":"2026-11-03T15:18:48.046Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"fb29de63-38ba-469a-bb0c-c61d5147092d","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:18:47Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:18:47Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:18:47Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:18:47Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:18:49.960Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"76641c28-666d-4bd5-8890-a86d8f5ce3c8"}
{"level":"info","ts":"2026-11-03T15:18:49.960Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"76641c28-666d-4bd5-8890-a86d8f5ce3c8"}
{"level":"info","ts":"2026-11-03T15:18:49.960Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"76641c28-666d-4bd5-8890-a86d8f5ce3c8","1 head service found":"counter-n8dh7-head-svc"}
{"level":"info","ts":"2026-11-03T15:18:49.960Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"76641c28-666d-4bd5-8890-a86d8f5ce3c8","Found 1 head Pod":"counter-n8dh7-head-cshcl","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:18:49.960Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"76641c28-666d-4bd5-8890-a86d8f5ce3c8","head Pod":"counter-n8dh7-head-cshcl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-n8dh7-head-cshcl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:18:49.960Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"76641c28-666d-4bd5-8890-a86d8f5ce3c8","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:18:49.960Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"76641c28-666d-4bd5-8890-a86d8f5ce3c8","worker Pod":"counter-n8dh7-small-group-worker-6tgfv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-n8dh7-small-group-worker-6tgfv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:18:49.960Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"76641c28-666d-4bd5-8890-a86d8f5ce3c8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:18:49.960Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"76641c28-666d-4bd5-8890-a86d8f5ce3c8","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:18:49.960Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"76641c28-666d-4bd5-8890-a86d8f5ce3c8","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:18:49.960Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"76641c28-666d-4bd5-8890-a86d8f5ce3c8","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:18:49.960Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"76641c28-666d-4bd5-8890-a86d8f5ce3c8","podName":"counter-n8dh7-small-group-worker-6tgfv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:18:49.960Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"76641c28-666d-4bd5-8890-a86d8f5ce3c8","podName":"counter-n8dh7-small-group-worker-6tgfv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:18:49.960Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"76641c28-666d-4bd5-8890-a86d8f5ce3c8","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:18:49.960Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"76641c28-666d-4bd5-8890-a86d8f5ce3c8","seconds":300}
{"level":"info","ts":"2026-11-03T15:18:51.047Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"ba62834f-2678-4b24-b167-469b54c624e7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:18:51.047Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"ba62834f-2678-4b24-b167-469b54c624e7","name":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:18:51.047Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"ba62834f-2678-4b24-b167-469b54c624e7","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:18:51.048Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"ba62834f-2678-4b24-b167-469b54c624e7"}
{"level":"info","ts":"2026-11-03T15:18:51.048Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"ba62834f-2678-4b24-b167-469b54c624e7","RayCluster":"counter-n8dh7","State":""}
{"level":"info","ts":"2026-11-03T15:18:51.048Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"ba62834f-2678-4b24-b167-469b54c624e7","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:18:47Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:18:47Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:18:47Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:18:47Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:18:51.794Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"825faa4d-2ea5-4219-a4ac-39350130d376"}
{"level":"info","ts":"2026-11-03T15:18:51.794Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"825faa4d-2ea5-4219-a4ac-39350130d376"}
{"level":"info","ts":"2026-11-03T15:18:51.794Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"825faa4d-2ea5-4219-a4ac-39350130d376","1 head service found":"counter-n8dh7-head-svc"}
{"level":"info","ts":"2026-11-03T15:18:51.794Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"825faa4d-2ea5-4219-a4ac-39350130d376","Found 1 head Pod":"counter-n8dh7-head-cshcl","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:18:51.794Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"825faa4d-2ea5-4219-a4ac-39350130d376","head Pod":"counter-n8dh7-head-cshcl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-n8dh7-head-cshcl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:18:51.794Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"825faa4d-2ea5-4219-a4ac-39350130d376","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:18:51.794Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"825faa4d-2ea5-4219-a4ac-39350130d376","worker Pod":"counter-n8dh7-small-group-worker-6tgfv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-n8dh7-small-group-worker-6tgfv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:18:51.794Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"825faa4d-2ea5-4219-a4ac-39350130d376","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:18:51.794Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"825faa4d-2ea5-4219-a4ac-39350130d376","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:18:51.794Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"825faa4d-2ea5-4219-a4ac-39350130d376","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:18:51.794Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"825faa4d-2ea5-4219-a4ac-39350130d376","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:18:51.794Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"825faa4d-2ea5-4219-a4ac-39350130d376","podName":"counter-n8dh7-small-group-worker-6tgfv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:18:51.795Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"825faa4d-2ea5-4219-a4ac-39350130d376","podName":"counter-n8dh7-small-group-worker-6tgfv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:18:51.795Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"825faa4d-2ea5-4219-a4ac-39350130d376","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:18:51.795Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"825faa4d-2ea5-4219-a4ac-39350130d376","seconds":300}
{"level":"info","ts":"2026-11-03T15:18:52.804Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"1f1bd027-adab-402c-b7a9-1e5ac77ab96a"}
{"level":"info","ts":"2026-11-03T15:18:52.804Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"1f1bd027-adab-402c-b7a9-1e5ac77ab96a"}
{"level":"info","ts":"2026-11-03T15:18:52.804Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"1f1bd027-adab-402c-b7a9-1e5ac77ab96a","1 head service found":"counter-n8dh7-head-svc"}
{"level":"info","ts":"2026-11-03T15:18:52.804Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"1f1bd027-adab-402c-b7a9-1e5ac77ab96a","Found 1 head Pod":"counter-n8dh7-head-cshcl","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:18:52.804Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"1f1bd027-adab-402c-b7a9-1e5ac77ab96a","head Pod":"counter-n8dh7-head-cshcl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-n8dh7-head-cshcl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:18:52.804Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"1f1bd027-adab-402c-b7a9-1e5ac77ab96a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:18:52.804Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"1f1bd027-adab-402c-b7a9-1e5ac77ab96a","worker Pod":"counter-n8dh7-small-group-worker-6tgfv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-n8dh7-small-group-worker-6tgfv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:18:52.804Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"1f1bd027-adab-402c-b7a9-1e5ac77ab96a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:18:52.804Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"1f1bd027-adab-402c-b7a9-1e5ac77ab96a","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:18:52.804Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"1f1bd027-adab-402c-b7a9-1e5ac77ab96a","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:18:52.804Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"1f1bd027-adab-402c-b7a9-1e5ac77ab96a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:18:52.804Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"1f1bd027-adab-402c-b7a9-1e5ac77ab96a","podName":"counter-n8dh7-small-group-worker-6tgfv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:18:52.804Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"1f1bd027-adab-402c-b7a9-1e5ac77ab96a","podName":"counter-n8dh7-small-group-worker-6tgfv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:18:52.804Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"1f1bd027-adab-402c-b7a9-1e5ac77ab96a","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:18:52.804Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"1f1bd027-adab-402c-b7a9-1e5ac77ab96a","seconds":300}
{"level":"info","ts":"2026-11-03T15:18:54.049Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"e5bb306b-c01b-4afb-a449-beeb0cc1e812","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:18:54.049Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"e5bb306b-c01b-4afb-a449-beeb0cc1e812","name":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:18:54.049Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"e5bb306b-c01b-4afb-a449-beeb0cc1e812","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:18:54.049Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"e5bb306b-c01b-4afb-a449-beeb0cc1e812"}
{"level":"info","ts":"2026-11-03T15:18:54.049Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"e5bb306b-c01b-4afb-a449-beeb0cc1e812","RayCluster":"counter-n8dh7","State":""}
{"level":"info","ts":"2026-11-03T15:18:54.049Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"e5bb306b-c01b-4afb-a449-beeb0cc1e812","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:18:47Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:18:47Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:18:47Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:18:47Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:18:57.050Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"cca75458-f15b-40a4-8d35-e6f06d20cd88","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:18:57.050Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"cca75458-f15b-40a4-8d35-e6f06d20cd88","name":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:18:57.050Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"cca75458-f15b-40a4-8d35-e6f06d20cd88","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:18:57.050Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"cca75458-f15b-40a4-8d35-e6f06d20cd88"}
{"level":"info","ts":"2026-11-03T15:18:57.050Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"cca75458-f15b-40a4-8d35-e6f06d20cd88","RayCluster":"counter-n8dh7","State":""}
{"level":"info","ts":"2026-11-03T15:18:57.050Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"cca75458-f15b-40a4-8d35-e6f06d20cd88","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:18:47Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:18:47Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:18:47Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:18:47Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:19:00.051Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"8ace25e1-0243-4e3a-809a-84269c028536","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:19:00.051Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"8ace25e1-0243-4e3a-809a-84269c028536","name":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:19:00.051Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"8ace25e1-0243-4e3a-809a-84269c028536","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:19:00.052Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"8ace25e1-0243-4e3a-809a-84269c028536"}
{"level":"info","ts":"2026-11-03T15:19:00.052Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"8ace25e1-0243-4e3a-809a-84269c028536","RayCluster":"counter-n8dh7","State":""}
{"level":"info","ts":"2026-11-03T15:19:00.052Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"8ace25e1-0243-4e3a-809a-84269c028536","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:18:47Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:18:47Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:18:47Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:18:47Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:19:01.858Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"960974da-91df-4464-b795-94a0329381ab"}
{"level":"info","ts":"2026-11-03T15:19:01.858Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"960974da-91df-4464-b795-94a0329381ab"}
{"level":"info","ts":"2026-11-03T15:19:01.858Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"960974da-91df-4464-b795-94a0329381ab","1 head service found":"counter-n8dh7-head-svc"}
{"level":"info","ts":"2026-11-03T15:19:01.859Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"960974da-91df-4464-b795-94a0329381ab","Found 1 head Pod":"counter-n8dh7-head-cshcl","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:19:01.859Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"960974da-91df-4464-b795-94a0329381ab","head Pod":"counter-n8dh7-head-cshcl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-n8dh7-head-cshcl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:19:01.859Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"960974da-91df-4464-b795-94a0329381ab","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:19:01.859Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"960974da-91df-4464-b795-94a0329381ab","worker Pod":"counter-n8dh7-small-group-worker-6tgfv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-n8dh7-small-group-worker-6tgfv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:19:01.859Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"960974da-91df-4464-b795-94a0329381ab","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:19:01.859Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"960974da-91df-4464-b795-94a0329381ab","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:19:01.859Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"960974da-91df-4464-b795-94a0329381ab","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:19:01.859Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"960974da-91df-4464-b795-94a0329381ab","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:19:01.859Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"960974da-91df-4464-b795-94a0329381ab","podName":"counter-n8dh7-small-group-worker-6tgfv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:19:01.859Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"960974da-91df-4464-b795-94a0329381ab","podName":"counter-n8dh7-small-group-worker-6tgfv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:19:01.859Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"960974da-91df-4464-b795-94a0329381ab","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:19:01.859Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"960974da-91df-4464-b795-94a0329381ab","seconds":300}
{"level":"info","ts":"2026-11-03T15:19:02.847Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"e1a1e49f-9785-4efc-85ce-0073ccc9bbf1"}
{"level":"info","ts":"2026-11-03T15:19:02.847Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"e1a1e49f-9785-4efc-85ce-0073ccc9bbf1"}
{"level":"info","ts":"2026-11-03T15:19:02.847Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"e1a1e49f-9785-4efc-85ce-0073ccc9bbf1","1 head service found":"counter-n8dh7-head-svc"}
{"level":"info","ts":"2026-11-03T15:19:02.847Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"e1a1e49f-9785-4efc-85ce-0073ccc9bbf1","Found 1 head Pod":"counter-n8dh7-head-cshcl","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:19:02.847Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"e1a1e49f-9785-4efc-85ce-0073ccc9bbf1","head Pod":"counter-n8dh7-head-cshcl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-n8dh7-head-cshcl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:19:02.847Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"e1a1e49f-9785-4efc-85ce-0073ccc9bbf1","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:19:02.847Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"e1a1e49f-9785-4efc-85ce-0073ccc9bbf1","worker Pod":"counter-n8dh7-small-group-worker-6tgfv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-n8dh7-small-group-worker-6tgfv. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:19:02.847Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"e1a1e49f-9785-4efc-85ce-0073ccc9bbf1","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:19:02.847Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"e1a1e49f-9785-4efc-85ce-0073ccc9bbf1","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:19:02.847Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"e1a1e49f-9785-4efc-85ce-0073ccc9bbf1","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:19:02.847Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"e1a1e49f-9785-4efc-85ce-0073ccc9bbf1","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:19:02.848Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"e1a1e49f-9785-4efc-85ce-0073ccc9bbf1","podName":"counter-n8dh7-small-group-worker-6tgfv","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:17:19Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:19:02.848Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"e1a1e49f-9785-4efc-85ce-0073ccc9bbf1","podName":"counter-n8dh7-small-group-worker-6tgfv","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:17:19Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:19:02.848Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"e1a1e49f-9785-4efc-85ce-0073ccc9bbf1","name":"counter-n8dh7","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:18:47Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:18:47Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:19:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:18:47Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:19:02.947Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"268af18a-9c0c-45f0-80b9-6cfe2acd3e4b","cluster":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:19:02.948Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"1d4400d2-df5f-481d-ade8-42dd34c67268","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:19:02.948Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"1d4400d2-df5f-481d-ade8-42dd34c67268","name":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:19:02.948Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"1d4400d2-df5f-481d-ade8-42dd34c67268","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:19:02.948Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"1d4400d2-df5f-481d-ade8-42dd34c67268"}
{"level":"info","ts":"2026-11-03T15:19:02.948Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"1d4400d2-df5f-481d-ade8-42dd34c67268","RayCluster":"counter-n8dh7","State":""}
{"level":"info","ts":"2026-11-03T15:19:02.948Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"1d4400d2-df5f-481d-ade8-42dd34c67268","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:18:47Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:18:47Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:19:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:18:47Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:19:02.948Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"1d4400d2-df5f-481d-ade8-42dd34c67268","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:19:03.048Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"e18be503-ecd8-42cc-b545-b00a3892d0b5","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:19:03.048Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"e18be503-ecd8-42cc-b545-b00a3892d0b5","name":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:19:03.048Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"e18be503-ecd8-42cc-b545-b00a3892d0b5","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:19:03.048Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"e18be503-ecd8-42cc-b545-b00a3892d0b5"}
{"level":"info","ts":"2026-11-03T15:19:03.048Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"e18be503-ecd8-42cc-b545-b00a3892d0b5","RayCluster":"counter-n8dh7","State":""}
{"level":"info","ts":"2026-11-03T15:19:03.048Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"e18be503-ecd8-42cc-b545-b00a3892d0b5","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:19:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:18:47Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:19:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:18:47Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:19:04.947Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"6424d0a5-9ac0-4fd8-8836-1c6b0fb39160"}
{"level":"info","ts":"2026-11-03T15:19:04.947Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"6424d0a5-9ac0-4fd8-8836-1c6b0fb39160"}
{"level":"info","ts":"2026-11-03T15:19:04.947Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"6424d0a5-9ac0-4fd8-8836-1c6b0fb39160","1 head service found":"counter-n8dh7-head-svc"}
{"level":"info","ts":"2026-11-03T15:19:04.948Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"6424d0a5-9ac0-4fd8-8836-1c6b0fb39160","Found 1 head Pod":"counter-n8dh7-head-cshcl","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:19:04.948Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"6424d0a5-9ac0-4fd8-8836-1c6b0fb39160","head Pod":"counter-n8dh7-head-cshcl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-n8dh7-head-cshcl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:19:04.948Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"6424d0a5-9ac0-4fd8-8836-1c6b0fb39160","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:19:04.948Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"6424d0a5-9ac0-4fd8-8836-1c6b0fb39160","worker Pod":"counter-n8dh7-small-group-worker-6tgfv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-n8dh7-small-group-worker-6tgfv. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:19:04.948Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"6424d0a5-9ac0-4fd8-8836-1c6b0fb39160","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:19:04.948Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"6424d0a5-9ac0-4fd8-8836-1c6b0fb39160","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:19:04.948Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"6424d0a5-9ac0-4fd8-8836-1c6b0fb39160","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:19:04.948Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"6424d0a5-9ac0-4fd8-8836-1c6b0fb39160","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:19:04.948Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"6424d0a5-9ac0-4fd8-8836-1c6b0fb39160","podName":"counter-n8dh7-small-group-worker-6tgfv","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:17:19Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:19:04.948Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"6424d0a5-9ac0-4fd8-8836-1c6b0fb39160","podName":"counter-n8dh7-small-group-worker-6tgfv","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:17:19Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:19:04.948Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"6424d0a5-9ac0-4fd8-8836-1c6b0fb39160","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:19:04.948Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"6424d0a5-9ac0-4fd8-8836-1c6b0fb39160","seconds":300}
{"level":"info","ts":"2026-11-03T15:19:06.050Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"7b1ac96c-4e4b-47fb-997c-56e1de31adf1","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:19:06.050Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"7b1ac96c-4e4b-47fb-997c-56e1de31adf1","name":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:19:06.050Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"7b1ac96c-4e4b-47fb-997c-56e1de31adf1","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:19:06.050Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"7b1ac96c-4e4b-47fb-997c-56e1de31adf1"}
{"level":"info","ts":"2026-11-03T15:19:06.050Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"7b1ac96c-4e4b-47fb-997c-56e1de31adf1","RayCluster":"counter-n8dh7","State":""}
{"level":"info","ts":"2026-11-03T15:19:06.050Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"7b1ac96c-4e4b-47fb-997c-56e1de31adf1","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:19:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:18:47Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:19:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:18:47Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:19:09.051Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"18df39e0-0bf6-494b-ad2f-ea7ea040e21a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:19:09.051Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"18df39e0-0bf6-494b-ad2f-ea7ea040e21a","name":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:19:09.051Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"18df39e0-0bf6-494b-ad2f-ea7ea040e21a","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:19:09.051Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"18df39e0-0bf6-494b-ad2f-ea7ea040e21a"}
{"level":"info","ts":"2026-11-03T15:19:09.051Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"18df39e0-0bf6-494b-ad2f-ea7ea040e21a","RayCluster":"counter-n8dh7","State":""}
{"level":"info","ts":"2026-11-03T15:19:09.051Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"18df39e0-0bf6-494b-ad2f-ea7ea040e21a","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:19:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:18:47Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:19:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:18:47Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:19:12.052Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"6ab076c5-a50b-4865-8b70-ffd93c2283d9","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:19:12.052Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"6ab076c5-a50b-4865-8b70-ffd93c2283d9","name":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:19:12.052Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"6ab076c5-a50b-4865-8b70-ffd93c2283d9","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:19:12.052Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"6ab076c5-a50b-4865-8b70-ffd93c2283d9"}
{"level":"info","ts":"2026-11-03T15:19:12.052Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"6ab076c5-a50b-4865-8b70-ffd93c2283d9","RayCluster":"counter-n8dh7","State":""}
{"level":"info","ts":"2026-11-03T15:19:12.052Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"6ab076c5-a50b-4865-8b70-ffd93c2283d9","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:19:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:18:47Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:19:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:18:47Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:19:13.857Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"c8901e51-3bf3-45f0-95da-b61ea41bf32e"}
{"level":"info","ts":"2026-11-03T15:19:13.857Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"c8901e51-3bf3-45f0-95da-b61ea41bf32e"}
{"level":"info","ts":"2026-11-03T15:19:13.857Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"c8901e51-3bf3-45f0-95da-b61ea41bf32e","1 head service found":"counter-n8dh7-head-svc"}
{"level":"info","ts":"2026-11-03T15:19:13.858Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"c8901e51-3bf3-45f0-95da-b61ea41bf32e","Found 1 head Pod":"counter-n8dh7-head-cshcl","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:19:13.858Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"c8901e51-3bf3-45f0-95da-b61ea41bf32e","head Pod":"counter-n8dh7-head-cshcl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-n8dh7-head-cshcl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:19:13.858Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"c8901e51-3bf3-45f0-95da-b61ea41bf32e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:19:13.858Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"c8901e51-3bf3-45f0-95da-b61ea41bf32e","worker Pod":"counter-n8dh7-small-group-worker-6tgfv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-n8dh7-small-group-worker-6tgfv. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:19:13.858Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"c8901e51-3bf3-45f0-95da-b61ea41bf32e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:19:13.858Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"c8901e51-3bf3-45f0-95da-b61ea41bf32e","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:19:13.858Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"c8901e51-3bf3-45f0-95da-b61ea41bf32e","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:19:13.858Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"c8901e51-3bf3-45f0-95da-b61ea41bf32e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:19:13.859Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"c8901e51-3bf3-45f0-95da-b61ea41bf32e","name":"counter-n8dh7","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:19:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:18:47Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:19:13Z","stateTransitionTimes":{"ready":"2026-11-03T15:19:13Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:18:47Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:19:13Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:19:13.924Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"95a19735-580f-47c4-87be-6d7ab3be0c43","cluster":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:19:13.924Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"1fe9cc06-07a4-4f1e-9233-a4008bfdc698","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:19:13.924Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"1fe9cc06-07a4-4f1e-9233-a4008bfdc698","name":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:19:13.924Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"1fe9cc06-07a4-4f1e-9233-a4008bfdc698","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:19:13.924Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"1fe9cc06-07a4-4f1e-9233-a4008bfdc698"}
{"level":"info","ts":"2026-11-03T15:19:14.001Z","logger":"controllers.RayJob","msg":"Created submitter Kubernetes Job for RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"1fe9cc06-07a4-4f1e-9233-a4008bfdc698","Kubernetes Job":"counter"}
{"level":"info","ts":"2026-11-03T15:19:14.001Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"1fe9cc06-07a4-4f1e-9233-a4008bfdc698","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:19:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:18:47Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","dashboardURL":"counter-n8dh7-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:19:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:18:47Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:19:14.001Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"1fe9cc06-07a4-4f1e-9233-a4008bfdc698","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-11-03T15:19:14.078Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"59a5d79a-3dca-4cee-a00e-6fd200cd66a6","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:19:14.078Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"59a5d79a-3dca-4cee-a00e-6fd200cd66a6","name":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:19:14.078Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"59a5d79a-3dca-4cee-a00e-6fd200cd66a6","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:19:14.078Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"59a5d79a-3dca-4cee-a00e-6fd200cd66a6"}
{"level":"info","ts":"2026-11-03T15:19:14.078Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job for RayJob already exists","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"59a5d79a-3dca-4cee-a00e-6fd200cd66a6","Kubernetes Job":"counter"}
{"level":"info","ts":"2026-11-03T15:19:14.078Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"59a5d79a-3dca-4cee-a00e-6fd200cd66a6","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:19:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:18:47Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","dashboardURL":"counter-n8dh7-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:19:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:18:47Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:19:14.078Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"59a5d79a-3dca-4cee-a00e-6fd200cd66a6","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-11-03T15:19:14.136Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"59a5d79a-3dca-4cee-a00e-6fd200cd66a6","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-11-03T15:19:14.136Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"59a5d79a-3dca-4cee-a00e-6fd200cd66a6"}
{"level":"error","ts":"2026-11-03T15:19:14.136Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"59a5d79a-3dca-4cee-a00e-6fd200cd66a6","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:19:14.136Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"be01a2e7-e71c-4191-af67-bded033fc7d1","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:19:14.136Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"be01a2e7-e71c-4191-af67-bded033fc7d1","name":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:19:14.136Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"be01a2e7-e71c-4191-af67-bded033fc7d1","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:19:14.136Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"be01a2e7-e71c-4191-af67-bded033fc7d1"}
{"level":"info","ts":"2026-11-03T15:19:14.136Z","logger":"controllers.RayJob","msg":"Create a Kubernetes Service","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"be01a2e7-e71c-4191-af67-bded033fc7d1","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:19:14.199Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"3816c214-8380-4247-b46f-522780491a04","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:19:14.199Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"3816c214-8380-4247-b46f-522780491a04","name":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:19:14.199Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"3816c214-8380-4247-b46f-522780491a04","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:19:14.200Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"3816c214-8380-4247-b46f-522780491a04"}
{"level":"info","ts":"2026-11-03T15:19:14.200Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"3816c214-8380-4247-b46f-522780491a04","rayCluster":"counter-n8dh7","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:19:15.945Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"8654f05a-c9fd-4859-a37c-31cca8757019"}
{"level":"info","ts":"2026-11-03T15:19:15.945Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"8654f05a-c9fd-4859-a37c-31cca8757019"}
{"level":"info","ts":"2026-11-03T15:19:15.945Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"8654f05a-c9fd-4859-a37c-31cca8757019","1 head service found":"counter-n8dh7-head-svc"}
{"level":"info","ts":"2026-11-03T15:19:15.945Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"8654f05a-c9fd-4859-a37c-31cca8757019","Found 1 head Pod":"counter-n8dh7-head-cshcl","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:19:15.945Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"8654f05a-c9fd-4859-a37c-31cca8757019","head Pod":"counter-n8dh7-head-cshcl","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-n8dh7-head-cshcl. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:19:15.945Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"8654f05a-c9fd-4859-a37c-31cca8757019","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:19:15.945Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"8654f05a-c9fd-4859-a37c-31cca8757019","worker Pod":"counter-n8dh7-small-group-worker-6tgfv","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-n8dh7-small-group-worker-6tgfv. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:19:15.945Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"8654f05a-c9fd-4859-a37c-31cca8757019","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:19:15.945Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"8654f05a-c9fd-4859-a37c-31cca8757019","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:19:15.945Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"8654f05a-c9fd-4859-a37c-31cca8757019","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:19:15.945Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"8654f05a-c9fd-4859-a37c-31cca8757019","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:19:15.946Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"8654f05a-c9fd-4859-a37c-31cca8757019","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:19:15.946Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"8654f05a-c9fd-4859-a37c-31cca8757019","seconds":300}
{"level":"info","ts":"2026-11-03T15:19:16.865Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"d6e78aec-168d-46e2-94e1-7851f803706c","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:19:16.865Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"d6e78aec-168d-46e2-94e1-7851f803706c","name":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:19:16.865Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"d6e78aec-168d-46e2-94e1-7851f803706c","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:19:16.865Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"d6e78aec-168d-46e2-94e1-7851f803706c"}
{"level":"info","ts":"2026-11-03T15:19:16.865Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"d6e78aec-168d-46e2-94e1-7851f803706c","rayCluster":"counter-n8dh7","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:19:19.872Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"b108e9c9-6ab6-4b97-8a16-370ab432a802","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:19:19.872Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"b108e9c9-6ab6-4b97-8a16-370ab432a802","name":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:19:19.872Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"b108e9c9-6ab6-4b97-8a16-370ab432a802","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:19:19.872Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"b108e9c9-6ab6-4b97-8a16-370ab432a802"}
{"level":"info","ts":"2026-11-03T15:19:19.872Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"b108e9c9-6ab6-4b97-8a16-370ab432a802","rayCluster":"counter-n8dh7","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:19:22.878Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"2f1572ac-e81b-4304-8eca-bdd8994fc3cb","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:19:22.878Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"2f1572ac-e81b-4304-8eca-bdd8994fc3cb","name":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:19:22.878Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"2f1572ac-e81b-4304-8eca-bdd8994fc3cb","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:19:22.878Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"2f1572ac-e81b-4304-8eca-bdd8994fc3cb"}
{"level":"info","ts":"2026-11-03T15:19:22.878Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"2f1572ac-e81b-4304-8eca-bdd8994fc3cb","rayCluster":"counter-n8dh7","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:19:22.884Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"2f1572ac-e81b-4304-8eca-bdd8994fc3cb","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","dashboardURL":"counter-n8dh7-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:19:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:18:47Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:19:22Z"},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","dashboardURL":"counter-n8dh7-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:19:13Z","stateTransitionTimes":{"ready":"2026-11-03T15:19:13Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:18:47Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:19:13Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:19:22.884Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"2f1572ac-e81b-4304-8eca-bdd8994fc3cb","old JobStatus":"","new JobStatus":"PENDING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-11-03T15:19:22.997Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"2f41a4cd-797a-4c8f-a315-bcd6090d9692","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:19:22.997Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"2f41a4cd-797a-4c8f-a315-bcd6090d9692","name":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:19:22.997Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"2f41a4cd-797a-4c8f-a315-bcd6090d9692","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:19:22.997Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"2f41a4cd-797a-4c8f-a315-bcd6090d9692"}
{"level":"info","ts":"2026-11-03T15:19:22.998Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"2f41a4cd-797a-4c8f-a315-bcd6090d9692","rayCluster":"counter-n8dh7","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:19:23.001Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"2f41a4cd-797a-4c8f-a315-bcd6090d9692","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:19:22Z"},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","dashboardURL":"counter-n8dh7-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:19:13Z","stateTransitionTimes":{"ready":"2026-11-03T15:19:13Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:18:47Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:19:13Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:19:22Z"},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","dashboardURL":"counter-n8dh7-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:19:13Z","stateTransitionTimes":{"ready":"2026-11-03T15:19:13Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:18:47Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:19:13Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:19:26.001Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"95243615-2509-4e91-bbb9-732b4ab60a5f","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:19:26.001Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"95243615-2509-4e91-bbb9-732b4ab60a5f","name":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:19:26.001Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"95243615-2509-4e91-bbb9-732b4ab60a5f","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:19:26.002Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"95243615-2509-4e91-bbb9-732b4ab60a5f"}
{"level":"info","ts":"2026-11-03T15:19:26.002Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"95243615-2509-4e91-bbb9-732b4ab60a5f","rayCluster":"counter-n8dh7","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:19:26.005Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"95243615-2509-4e91-bbb9-732b4ab60a5f","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:19:22Z"},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","dashboardURL":"counter-n8dh7-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:19:13Z","stateTransitionTimes":{"ready":"2026-11-03T15:19:13Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:18:47Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:19:13Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:19:22Z"},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","dashboardURL":"counter-n8dh7-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:19:13Z","stateTransitionTimes":{"ready":"2026-11-03T15:19:13Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:18:47Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:19:13Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:19:26.005Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"95243615-2509-4e91-bbb9-732b4ab60a5f","old JobStatus":"PENDING","new JobStatus":"RUNNING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-11-03T15:19:26.106Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"b8b82cd5-8f12-468e-8a8e-32ec9af7c134","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:19:26.106Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"b8b82cd5-8f12-468e-8a8e-32ec9af7c134","name":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:19:26.106Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"b8b82cd5-8f12-468e-8a8e-32ec9af7c134","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:19:26.106Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"b8b82cd5-8f12-468e-8a8e-32ec9af7c134"}
{"level":"info","ts":"2026-11-03T15:19:26.106Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"b8b82cd5-8f12-468e-8a8e-32ec9af7c134","rayCluster":"counter-n8dh7","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:19:26.109Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"b8b82cd5-8f12-468e-8a8e-32ec9af7c134","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:19:22Z"},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","dashboardURL":"counter-n8dh7-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:19:13Z","stateTransitionTimes":{"ready":"2026-11-03T15:19:13Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:18:47Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:19:13Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:19:22Z"},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","dashboardURL":"counter-n8dh7-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:19:13Z","stateTransitionTimes":{"ready":"2026-11-03T15:19:13Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:18:47Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:19:13Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:19:29.110Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"063f9af3-d7a5-4030-8e72-93ef8cf20d6a","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:19:29.110Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"063f9af3-d7a5-4030-8e72-93ef8cf20d6a","name":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:19:29.110Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"063f9af3-d7a5-4030-8e72-93ef8cf20d6a","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:19:29.110Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"063f9af3-d7a5-4030-8e72-93ef8cf20d6a"}
{"level":"info","ts":"2026-11-03T15:19:29.110Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"063f9af3-d7a5-4030-8e72-93ef8cf20d6a","rayCluster":"counter-n8dh7","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:19:29.114Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"063f9af3-d7a5-4030-8e72-93ef8cf20d6a","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:19:22Z"},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","dashboardURL":"counter-n8dh7-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:19:13Z","stateTransitionTimes":{"ready":"2026-11-03T15:19:13Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:18:47Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:19:13Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:19:22Z","endTime":"2026-11-03T15:19:26Z"},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","dashboardURL":"counter-n8dh7-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:19:13Z","stateTransitionTimes":{"ready":"2026-11-03T15:19:13Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:18:47Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:19:13Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:19:29.114Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"063f9af3-d7a5-4030-8e72-93ef8cf20d6a","old JobStatus":"RUNNING","new JobStatus":"SUCCEEDED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-11-03T15:19:29.214Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"b552b61e-9a34-46db-83f9-998cda6349bf","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:19:29.214Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"b552b61e-9a34-46db-83f9-998cda6349bf","name":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:19:29.215Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"b552b61e-9a34-46db-83f9-998cda6349bf","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:19:29.215Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"b552b61e-9a34-46db-83f9-998cda6349bf"}
{"level":"info","ts":"2026-11-03T15:19:29.215Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"b552b61e-9a34-46db-83f9-998cda6349bf","rayCluster":"counter-n8dh7","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:19:29.218Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"b552b61e-9a34-46db-83f9-998cda6349bf","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:19:22Z","endTime":"2026-11-03T15:19:26Z"},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","dashboardURL":"counter-n8dh7-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:19:13Z","stateTransitionTimes":{"ready":"2026-11-03T15:19:13Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:18:47Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:19:13Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:19:22Z","endTime":"2026-11-03T15:19:26Z"},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","dashboardURL":"counter-n8dh7-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:19:13Z","stateTransitionTimes":{"ready":"2026-11-03T15:19:13Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:18:47Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:19:13Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:19:31.933Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"71e52748-1c23-47a4-88f2-a3d78707287b","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:19:31.933Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"71e52748-1c23-47a4-88f2-a3d78707287b","name":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:19:31.933Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"71e52748-1c23-47a4-88f2-a3d78707287b","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:19:31.933Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"71e52748-1c23-47a4-88f2-a3d78707287b"}
{"level":"info","ts":"2026-11-03T15:19:31.933Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"71e52748-1c23-47a4-88f2-a3d78707287b","rayCluster":"counter-n8dh7","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:19:31.937Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"71e52748-1c23-47a4-88f2-a3d78707287b","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:19:22Z","endTime":"2026-11-03T15:19:26Z"},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","dashboardURL":"counter-n8dh7-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:19:13Z","stateTransitionTimes":{"ready":"2026-11-03T15:19:13Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:18:47Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:19:13Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:19:22Z","endTime":"2026-11-03T15:19:26Z"},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","dashboardURL":"counter-n8dh7-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:19:13Z","stateTransitionTimes":{"ready":"2026-11-03T15:19:13Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:18:47Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:19:13Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:19:32.941Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"07be0d6e-c2fe-4973-ab62-d39efcf92ddd","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:19:32.941Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"07be0d6e-c2fe-4973-ab62-d39efcf92ddd","name":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:19:32.941Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"07be0d6e-c2fe-4973-ab62-d39efcf92ddd","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:19:32.941Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"07be0d6e-c2fe-4973-ab62-d39efcf92ddd"}
{"level":"info","ts":"2026-11-03T15:19:32.941Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"07be0d6e-c2fe-4973-ab62-d39efcf92ddd","rayCluster":"counter-n8dh7","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:19:32.945Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"07be0d6e-c2fe-4973-ab62-d39efcf92ddd","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:19:22Z","endTime":"2026-11-03T15:19:26Z"},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","dashboardURL":"counter-n8dh7-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:19:13Z","stateTransitionTimes":{"ready":"2026-11-03T15:19:13Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:18:47Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:19:13Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:19:22Z","endTime":"2026-11-03T15:19:26Z"},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","dashboardURL":"counter-n8dh7-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:19:13Z","stateTransitionTimes":{"ready":"2026-11-03T15:19:13Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:18:47Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:19:13Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:19:32.961Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"9550bc0c-8bd3-48d0-9b36-ace4a005fe8f","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:19:32.961Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"9550bc0c-8bd3-48d0-9b36-ace4a005fe8f","name":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:19:32.961Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"9550bc0c-8bd3-48d0-9b36-ace4a005fe8f","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:19:32.961Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"9550bc0c-8bd3-48d0-9b36-ace4a005fe8f"}
{"level":"info","ts":"2026-11-03T15:19:32.961Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"9550bc0c-8bd3-48d0-9b36-ace4a005fe8f","rayCluster":"counter-n8dh7","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:19:32.963Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"9550bc0c-8bd3-48d0-9b36-ace4a005fe8f","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:19:22Z","endTime":"2026-11-03T15:19:26Z"},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","dashboardURL":"counter-n8dh7-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-11-03T15:17:15Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:19:13Z","stateTransitionTimes":{"ready":"2026-11-03T15:19:13Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:18:47Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:19:13Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:19:22Z","endTime":"2026-11-03T15:19:26Z"},"jobId":"counter-cvqlx","rayClusterName":"counter-n8dh7","dashboardURL":"counter-n8dh7-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Complete","message":"Job finished successfully.","startTime":"2026-11-03T15:17:15Z","succeeded":1,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:19:13Z","stateTransitionTimes":{"ready":"2026-11-03T15:19:13Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.38","serviceIP":"10.132.0.38","podName":"counter-n8dh7-head-cshcl","serviceName":"counter-n8dh7-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:18:47Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:19:13Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:17:15Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:17:15Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:19:32.964Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"9550bc0c-8bd3-48d0-9b36-ace4a005fe8f","old JobStatus":"SUCCEEDED","new JobStatus":"SUCCEEDED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Complete"}
{"level":"info","ts":"2026-11-03T15:19:33.064Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"c3ee3416-ac53-4145-9399-99933da77793","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:19:33.064Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"c3ee3416-ac53-4145-9399-99933da77793","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Complete","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-11-03 15:19:32 +0000 UTC","Now":"2026-11-03T15:19:33.064Z","ShutdownTime":"2026-11-03T15:19:32.000Z"}
{"level":"info","ts":"2026-11-03T15:19:33.124Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob is deleted","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"c3ee3416-ac53-4145-9399-99933da77793","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:19:33.124Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"c3ee3416-ac53-4145-9399-99933da77793","isClusterDeleted":false}
{"level":"info","ts":"2026-11-03T15:19:33.124Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"c3ee3416-ac53-4145-9399-99933da77793","deletionMechanism":"ShutdownAfterJobFinishes","RayCluster":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:19:33.125Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"d6470941-9863-42e0-8f0f-3c6fd46b21ca"}
{"level":"info","ts":"2026-11-03T15:19:33.125Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"counter-n8dh7","reconcileID":"e2dc432b-4829-43b3-aa4f-81ec9feb7ae0","namespacedName":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:19:33.125Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"counter-n8dh7","reconcileID":"e2dc432b-4829-43b3-aa4f-81ec9feb7ae0","cluster":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:19:33.125Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"counter-n8dh7","reconcileID":"e2dc432b-4829-43b3-aa4f-81ec9feb7ae0","cluster":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:19:33.125Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"1cc9c135-c09f-47e7-8d6b-6031d61df1db","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:19:33.125Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"counter-n8dh7","reconcileID":"e2dc432b-4829-43b3-aa4f-81ec9feb7ae0","namespace":"test-ns-kbx4w","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-11-03T15:19:33.125Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"1cc9c135-c09f-47e7-8d6b-6031d61df1db","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Complete","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-11-03 15:19:32 +0000 UTC","Now":"2026-11-03T15:19:33.125Z","ShutdownTime":"2026-11-03T15:19:32.000Z"}
{"level":"info","ts":"2026-11-03T15:19:33.125Z","logger":"controllers.RayJob","msg":"The deletion of the associated RayCluster for RayJob is ongoing.","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"1cc9c135-c09f-47e7-8d6b-6031d61df1db","RayCluster":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:19:33.125Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"1cc9c135-c09f-47e7-8d6b-6031d61df1db","isClusterDeleted":false}
{"level":"info","ts":"2026-11-03T15:19:33.125Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"1cc9c135-c09f-47e7-8d6b-6031d61df1db","deletionMechanism":"ShutdownAfterJobFinishes","RayCluster":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:19:33.182Z","logger":"authentication-controller","msg":"Deleted shared ReferenceGrant (last cluster with auth)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"counter-n8dh7","reconcileID":"e2dc432b-4829-43b3-aa4f-81ec9feb7ae0","name":"kuberay-gateway-access","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:19:33.237Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"counter-n8dh7","reconcileID":"e2dc432b-4829-43b3-aa4f-81ec9feb7ae0","name":"test-ns-kbx4w-counter-n8dh7","namespace":"opendatahub"}
{"level":"info","ts":"2026-11-03T15:19:33.290Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"counter-n8dh7","reconcileID":"e2dc432b-4829-43b3-aa4f-81ec9feb7ae0","configMap":"kube-rbac-proxy-config-counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:19:33.344Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"counter-n8dh7","reconcileID":"e2dc432b-4829-43b3-aa4f-81ec9feb7ae0","serviceAccount":"counter-n8dh7-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-11-03T15:19:33.509Z","logger":"raycluster-resource","msg":"default","name":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:19:33.509Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"counter-n8dh7","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:19:33.673Z","logger":"raycluster-resource","msg":"validate update","name":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:19:33.784Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"reconcileID":"85d81a87-b8a7-4d1c-87fb-882a24f71dec"}
{"level":"info","ts":"2026-11-03T15:19:33.784Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"2dfac69b-1bd3-430f-99a8-ad1617217c9a","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:19:33.784Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"2dfac69b-1bd3-430f-99a8-ad1617217c9a","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Complete","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-11-03 15:19:32 +0000 UTC","Now":"2026-11-03T15:19:33.784Z","ShutdownTime":"2026-11-03T15:19:32.000Z"}
{"level":"info","ts":"2026-11-03T15:19:33.784Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"2dfac69b-1bd3-430f-99a8-ad1617217c9a","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:19:33.784Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"2dfac69b-1bd3-430f-99a8-ad1617217c9a","isClusterDeleted":true}
{"level":"info","ts":"2026-11-03T15:19:33.784Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"2dfac69b-1bd3-430f-99a8-ad1617217c9a","deletionMechanism":"ShutdownAfterJobFinishes","RayCluster":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:19:33.785Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"counter-n8dh7","reconcileID":"e2dc432b-4829-43b3-aa4f-81ec9feb7ae0","cluster":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:19:33.785Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"counter-n8dh7","reconcileID":"047ffd78-0eb0-4424-91da-fed5ecbb2bba","namespacedName":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:19:33.785Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"counter-n8dh7","reconcileID":"047ffd78-0eb0-4424-91da-fed5ecbb2bba"}
{"level":"info","ts":"2026-11-03T15:19:33.785Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"counter-n8dh7","reconcileID":"047ffd78-0eb0-4424-91da-fed5ecbb2bba","namespace":"test-ns-kbx4w","authClusters":0}
{"level":"info","ts":"2026-11-03T15:19:33.785Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"counter-n8dh7","reconcileID":"047ffd78-0eb0-4424-91da-fed5ecbb2bba","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:19:33.787Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"counter-n8dh7","reconcileID":"ff2bc3ce-946c-4b22-ad67-b716c8bfad69","namespacedName":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:19:33.787Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"counter-n8dh7","reconcileID":"ff2bc3ce-946c-4b22-ad67-b716c8bfad69"}
{"level":"info","ts":"2026-11-03T15:19:33.787Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"counter-n8dh7","reconcileID":"ff2bc3ce-946c-4b22-ad67-b716c8bfad69","namespace":"test-ns-kbx4w","authClusters":0}
{"level":"info","ts":"2026-11-03T15:19:33.787Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"counter-n8dh7","reconcileID":"ff2bc3ce-946c-4b22-ad67-b716c8bfad69","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:19:34.695Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"3d067366-38dd-4b2f-ba9c-9dd5e26ecaa7","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:19:34.695Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"3d067366-38dd-4b2f-ba9c-9dd5e26ecaa7","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Complete","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-11-03 15:19:32 +0000 UTC","Now":"2026-11-03T15:19:34.695Z","ShutdownTime":"2026-11-03T15:19:32.000Z"}
{"level":"info","ts":"2026-11-03T15:19:34.695Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"3d067366-38dd-4b2f-ba9c-9dd5e26ecaa7","RayCluster":{"name":"counter-n8dh7","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:19:34.695Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"3d067366-38dd-4b2f-ba9c-9dd5e26ecaa7","isClusterDeleted":true}
{"level":"info","ts":"2026-11-03T15:19:34.695Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"3d067366-38dd-4b2f-ba9c-9dd5e26ecaa7","deletionMechanism":"ShutdownAfterJobFinishes","RayCluster":"counter-n8dh7"}
{"level":"info","ts":"2026-11-03T15:20:04.720Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"113b8857-ccc0-44b4-bd85-ad50aa4d064e","DeletionTimestamp":"2026-11-03 15:20:04 +0000 UTC"}
{"level":"info","ts":"2026-11-03T15:20:04.720Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"113b8857-ccc0-44b4-bd85-ad50aa4d064e","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-11-03T15:20:04.791Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"1ceb4cfb-1530-4473-85f9-f55e55839a44","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:20:04.792Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"1ceb4cfb-1530-4473-85f9-f55e55839a44","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-11-03T15:20:04.858Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"1ceb4cfb-1530-4473-85f9-f55e55839a44"}
{"level":"info","ts":"2026-11-03T15:20:04.858Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"1ceb4cfb-1530-4473-85f9-f55e55839a44","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-11-03T15:20:04.858Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"1ceb4cfb-1530-4473-85f9-f55e55839a44","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:20:04.858Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"1ceb4cfb-1530-4473-85f9-f55e55839a44","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:20:04.919Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"counter","namespace":"test-ns-kbx4w"},"reconcileID":"d2737005-6a43-4df6-880f-f852edd1b37f"}
{"level":"info","ts":"2026-11-03T15:20:04.919Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"f44e395f-52ca-4b66-ae48-a727e228e331","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:20:04.919Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"f44e395f-52ca-4b66-ae48-a727e228e331","name":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:20:04.919Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"f44e395f-52ca-4b66-ae48-a727e228e331","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:20:04.919Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"f44e395f-52ca-4b66-ae48-a727e228e331","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:20:04.970Z","logger":"raycluster-resource","msg":"default","name":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:20:04.970Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"fail-sgjqb","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:20:05.028Z","logger":"raycluster-resource","msg":"validate create","name":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:20:05.084Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"f44e395f-52ca-4b66-ae48-a727e228e331","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:20:05.084Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"b23d61fa-551f-41fc-b333-387dd632b997"}
{"level":"info","ts":"2026-11-03T15:20:05.084Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"b23d61fa-551f-41fc-b333-387dd632b997"}
{"level":"info","ts":"2026-11-03T15:20:05.084Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"f44e395f-52ca-4b66-ae48-a727e228e331"}
{"level":"info","ts":"2026-11-03T15:20:05.084Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"f44e395f-52ca-4b66-ae48-a727e228e331","RayCluster":"fail-sgjqb","State":""}
{"level":"info","ts":"2026-11-03T15:20:05.084Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"f44e395f-52ca-4b66-ae48-a727e228e331","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:20:05.084Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"354375ef-d0bd-4667-9b64-1274cab8ed77","cluster":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:20:05.084Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"fail-sgjqb","reconcileID":"383b58a4-92e2-4c7c-9a93-8832ec607889","namespacedName":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:20:05.084Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"fail-sgjqb","reconcileID":"383b58a4-92e2-4c7c-9a93-8832ec607889","mode":"IntegratedOAuth","cluster":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:20:05.085Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"fail-sgjqb","reconcileID":"383b58a4-92e2-4c7c-9a93-8832ec607889","cluster":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:20:05.085Z","logger":"controllers.NetworkPolicy","msg":"Reconciling NetworkPolicies for RayCluster","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"b5d2cea3-d3b7-4269-8262-6c9fe3b02064","cluster":"fail-sgjqb","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:20:05.085Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"fail-sgjqb","reconcileID":"383b58a4-92e2-4c7c-9a93-8832ec607889","cluster":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:20:05.085Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"5de15c58-152d-483b-acf4-8d382081a64e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:20:05.085Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"5de15c58-152d-483b-acf4-8d382081a64e","name":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:20:05.085Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"5de15c58-152d-483b-acf4-8d382081a64e","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:20:05.085Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"5de15c58-152d-483b-acf4-8d382081a64e"}
{"level":"info","ts":"2026-11-03T15:20:05.085Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"5de15c58-152d-483b-acf4-8d382081a64e","RayCluster":"fail-sgjqb","State":""}
{"level":"info","ts":"2026-11-03T15:20:05.085Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"5de15c58-152d-483b-acf4-8d382081a64e","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:20:05.136Z","logger":"raycluster-resource","msg":"default","name":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:20:05.136Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"fail-sgjqb","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:20:05.140Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"b23d61fa-551f-41fc-b333-387dd632b997","name":"fail-sgjqb-head-svc"}
{"level":"info","ts":"2026-11-03T15:20:05.140Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"b23d61fa-551f-41fc-b333-387dd632b997"}
{"level":"info","ts":"2026-11-03T15:20:05.140Z","logger":"controllers.NetworkPolicy","msg":"Successfully created NetworkPolicy","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"b5d2cea3-d3b7-4269-8262-6c9fe3b02064","name":"fail-sgjqb-head"}
{"level":"info","ts":"2026-11-03T15:20:05.187Z","logger":"raycluster-resource","msg":"validate update","name":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:20:05.192Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"b23d61fa-551f-41fc-b333-387dd632b997","error":"certificate ray-head-cert-fail-sgjqb not found: certificates.cert-manager.io \"ray-head-cert-fail-sgjqb\" not found"}
{"level":"error","ts":"2026-11-03T15:20:05.192Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"b23d61fa-551f-41fc-b333-387dd632b997","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-fail-sgjqb not found: certificates.cert-manager.io \"ray-head-cert-fail-sgjqb\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:20:05.192Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"b23d61fa-551f-41fc-b333-387dd632b997","error":"mTLS secrets not ready: certificate ray-head-cert-fail-sgjqb not found: certificates.cert-manager.io \"ray-head-cert-fail-sgjqb\" not found"}
{"level":"info","ts":"2026-11-03T15:20:05.192Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"b23d61fa-551f-41fc-b333-387dd632b997","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"fail-sgjqb","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:20:05.192Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"b23d61fa-551f-41fc-b333-387dd632b997","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"fail-sgjqb","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:20:05.192Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"b23d61fa-551f-41fc-b333-387dd632b997","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"fail-sgjqb","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:20:05.192Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"b23d61fa-551f-41fc-b333-387dd632b997","name":"fail-sgjqb","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:20:05.201Z","logger":"controllers.NetworkPolicy","msg":"Successfully created NetworkPolicy","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"b5d2cea3-d3b7-4269-8262-6c9fe3b02064","name":"fail-sgjqb-workers-small-group"}
{"level":"info","ts":"2026-11-03T15:20:05.201Z","logger":"controllers.NetworkPolicy","msg":"Successfully reconciled NetworkPolicies for RayCluster","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"b5d2cea3-d3b7-4269-8262-6c9fe3b02064","cluster":"fail-sgjqb","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:20:05.202Z","logger":"controllers.NetworkPolicy","msg":"Reconciling NetworkPolicies for RayCluster","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"63ed57b0-39c6-4774-a58c-09bc59f68470","cluster":"fail-sgjqb","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:20:05.243Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"da90a9e0-0651-48fb-97bb-50af5892241c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:20:05.243Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"da90a9e0-0651-48fb-97bb-50af5892241c","name":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:20:05.243Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"da90a9e0-0651-48fb-97bb-50af5892241c","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:20:05.244Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"da90a9e0-0651-48fb-97bb-50af5892241c"}
{"level":"info","ts":"2026-11-03T15:20:05.244Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"da90a9e0-0651-48fb-97bb-50af5892241c","RayCluster":"fail-sgjqb","State":""}
{"level":"info","ts":"2026-11-03T15:20:05.244Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"da90a9e0-0651-48fb-97bb-50af5892241c","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:20:05.247Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"b23d61fa-551f-41fc-b333-387dd632b997","name":"fail-sgjqb","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-sgjqb\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-kbx4w","name":"fail-sgjqb"}}
{"level":"info","ts":"2026-11-03T15:20:05.247Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"b23d61fa-551f-41fc-b333-387dd632b997"}
{"level":"error","ts":"2026-11-03T15:20:05.247Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"b23d61fa-551f-41fc-b333-387dd632b997","error":"mTLS secrets not ready: certificate ray-head-cert-fail-sgjqb not found: certificates.cert-manager.io \"ray-head-cert-fail-sgjqb\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:20:05.247Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"f6c054fe-e73f-4431-9fcf-0467cc4d1c95"}
{"level":"info","ts":"2026-11-03T15:20:05.248Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"f6c054fe-e73f-4431-9fcf-0467cc4d1c95"}
{"level":"info","ts":"2026-11-03T15:20:05.248Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"f6c054fe-e73f-4431-9fcf-0467cc4d1c95","1 head service found":"fail-sgjqb-head-svc"}
{"level":"info","ts":"2026-11-03T15:20:05.248Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"f6c054fe-e73f-4431-9fcf-0467cc4d1c95"}
{"level":"info","ts":"2026-11-03T15:20:05.298Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"fail-sgjqb","reconcileID":"383b58a4-92e2-4c7c-9a93-8832ec607889","name":"fail-sgjqb-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-11-03T15:20:05.299Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"f6c054fe-e73f-4431-9fcf-0467cc4d1c95","error":"certificate ray-head-cert-fail-sgjqb not found: certificates.cert-manager.io \"ray-head-cert-fail-sgjqb\" not found"}
{"level":"error","ts":"2026-11-03T15:20:05.299Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"f6c054fe-e73f-4431-9fcf-0467cc4d1c95","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-fail-sgjqb not found: certificates.cert-manager.io \"ray-head-cert-fail-sgjqb\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:20:05.299Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"f6c054fe-e73f-4431-9fcf-0467cc4d1c95","error":"mTLS secrets not ready: certificate ray-head-cert-fail-sgjqb not found: certificates.cert-manager.io \"ray-head-cert-fail-sgjqb\" not found"}
{"level":"info","ts":"2026-11-03T15:20:05.299Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"f6c054fe-e73f-4431-9fcf-0467cc4d1c95","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"fail-sgjqb","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:20:05.299Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"f6c054fe-e73f-4431-9fcf-0467cc4d1c95","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"fail-sgjqb","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:20:05.299Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"f6c054fe-e73f-4431-9fcf-0467cc4d1c95","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"fail-sgjqb","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:20:05.299Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"f6c054fe-e73f-4431-9fcf-0467cc4d1c95","name":"fail-sgjqb","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:20:05.328Z","logger":"controllers.NetworkPolicy","msg":"Successfully reconciled NetworkPolicies for RayCluster","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"63ed57b0-39c6-4774-a58c-09bc59f68470","cluster":"fail-sgjqb","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:20:05.354Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"fail-sgjqb","reconcileID":"383b58a4-92e2-4c7c-9a93-8832ec607889","name":"kuberay-gateway-access","namespace":"test-ns-kbx4w","operation":"created"}
{"level":"info","ts":"2026-11-03T15:20:05.383Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"f6c054fe-e73f-4431-9fcf-0467cc4d1c95"}
{"level":"error","ts":"2026-11-03T15:20:05.383Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"f6c054fe-e73f-4431-9fcf-0467cc4d1c95","error":"mTLS secrets not ready: certificate ray-head-cert-fail-sgjqb not found: certificates.cert-manager.io \"ray-head-cert-fail-sgjqb\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:20:05.384Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"095e4c9c-eb58-4898-8423-84c274e8e181","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:20:05.384Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"095e4c9c-eb58-4898-8423-84c274e8e181","name":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:20:05.384Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"095e4c9c-eb58-4898-8423-84c274e8e181","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:20:05.384Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"095e4c9c-eb58-4898-8423-84c274e8e181"}
{"level":"info","ts":"2026-11-03T15:20:05.384Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"095e4c9c-eb58-4898-8423-84c274e8e181","RayCluster":"fail-sgjqb","State":""}
{"level":"info","ts":"2026-11-03T15:20:05.384Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"095e4c9c-eb58-4898-8423-84c274e8e181","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:20:05.384Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"095e4c9c-eb58-4898-8423-84c274e8e181","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:20:05.394Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"44448a56-428b-45af-ab98-3eac2c73c9d8"}
{"level":"info","ts":"2026-11-03T15:20:05.394Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"44448a56-428b-45af-ab98-3eac2c73c9d8"}
{"level":"info","ts":"2026-11-03T15:20:05.394Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"44448a56-428b-45af-ab98-3eac2c73c9d8","1 head service found":"fail-sgjqb-head-svc"}
{"level":"info","ts":"2026-11-03T15:20:05.394Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"44448a56-428b-45af-ab98-3eac2c73c9d8"}
{"level":"info","ts":"2026-11-03T15:20:05.410Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"fail-sgjqb","reconcileID":"383b58a4-92e2-4c7c-9a93-8832ec607889","name":"test-ns-kbx4w-fail-sgjqb","operation":"created"}
{"level":"info","ts":"2026-11-03T15:20:05.446Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"44448a56-428b-45af-ab98-3eac2c73c9d8","error":"certificate ray-head-cert-fail-sgjqb not found: certificates.cert-manager.io \"ray-head-cert-fail-sgjqb\" not found"}
{"level":"error","ts":"2026-11-03T15:20:05.446Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"44448a56-428b-45af-ab98-3eac2c73c9d8","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-fail-sgjqb not found: certificates.cert-manager.io \"ray-head-cert-fail-sgjqb\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:20:05.446Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"44448a56-428b-45af-ab98-3eac2c73c9d8","error":"mTLS secrets not ready: certificate ray-head-cert-fail-sgjqb not found: certificates.cert-manager.io \"ray-head-cert-fail-sgjqb\" not found"}
{"level":"info","ts":"2026-11-03T15:20:05.446Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"44448a56-428b-45af-ab98-3eac2c73c9d8","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"fail-sgjqb","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:20:05.446Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"44448a56-428b-45af-ab98-3eac2c73c9d8","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"fail-sgjqb","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:20:05.446Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"44448a56-428b-45af-ab98-3eac2c73c9d8","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"fail-sgjqb","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:20:05.446Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"44448a56-428b-45af-ab98-3eac2c73c9d8"}
{"level":"error","ts":"2026-11-03T15:20:05.446Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"44448a56-428b-45af-ab98-3eac2c73c9d8","error":"mTLS secrets not ready: certificate ray-head-cert-fail-sgjqb not found: certificates.cert-manager.io \"ray-head-cert-fail-sgjqb\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:20:05.446Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"9788b109-2f3c-4128-afa8-04823553f988","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:20:05.447Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"9788b109-2f3c-4128-afa8-04823553f988","name":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:20:05.447Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"9788b109-2f3c-4128-afa8-04823553f988","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:20:05.447Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"9788b109-2f3c-4128-afa8-04823553f988"}
{"level":"info","ts":"2026-11-03T15:20:05.447Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"9788b109-2f3c-4128-afa8-04823553f988","RayCluster":"fail-sgjqb","State":""}
{"level":"info","ts":"2026-11-03T15:20:05.447Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"9788b109-2f3c-4128-afa8-04823553f988","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:20:05.464Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"fail-sgjqb","reconcileID":"383b58a4-92e2-4c7c-9a93-8832ec607889","name":"kube-rbac-proxy-config-fail-sgjqb","operation":"created"}
{"level":"info","ts":"2026-11-03T15:20:05.467Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"fed9d28d-c05a-4292-b20d-779ac28a897e"}
{"level":"info","ts":"2026-11-03T15:20:05.467Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"fed9d28d-c05a-4292-b20d-779ac28a897e"}
{"level":"info","ts":"2026-11-03T15:20:05.467Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"fed9d28d-c05a-4292-b20d-779ac28a897e","1 head service found":"fail-sgjqb-head-svc"}
{"level":"info","ts":"2026-11-03T15:20:05.467Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"fed9d28d-c05a-4292-b20d-779ac28a897e"}
{"level":"info","ts":"2026-11-03T15:20:05.518Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"fed9d28d-c05a-4292-b20d-779ac28a897e","error":"certificate ray-head-cert-fail-sgjqb not found: certificates.cert-manager.io \"ray-head-cert-fail-sgjqb\" not found"}
{"level":"error","ts":"2026-11-03T15:20:05.518Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"fed9d28d-c05a-4292-b20d-779ac28a897e","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-fail-sgjqb not found: certificates.cert-manager.io \"ray-head-cert-fail-sgjqb\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:20:05.518Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"fed9d28d-c05a-4292-b20d-779ac28a897e","error":"mTLS secrets not ready: certificate ray-head-cert-fail-sgjqb not found: certificates.cert-manager.io \"ray-head-cert-fail-sgjqb\" not found"}
{"level":"info","ts":"2026-11-03T15:20:05.518Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"fed9d28d-c05a-4292-b20d-779ac28a897e","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"fail-sgjqb","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:20:05.518Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"fed9d28d-c05a-4292-b20d-779ac28a897e","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"fail-sgjqb","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:20:05.518Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"fed9d28d-c05a-4292-b20d-779ac28a897e","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"fail-sgjqb","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:20:05.518Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"fed9d28d-c05a-4292-b20d-779ac28a897e"}
{"level":"error","ts":"2026-11-03T15:20:05.518Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"fed9d28d-c05a-4292-b20d-779ac28a897e","error":"mTLS secrets not ready: certificate ray-head-cert-fail-sgjqb not found: certificates.cert-manager.io \"ray-head-cert-fail-sgjqb\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-03T15:20:05.521Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"fail-sgjqb","reconcileID":"383b58a4-92e2-4c7c-9a93-8832ec607889","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"fail-sgjqb\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:167\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:20:05.521Z","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"fail-sgjqb","reconcileID":"383b58a4-92e2-4c7c-9a93-8832ec607889"}
{"level":"error","ts":"2026-11-03T15:20:05.521Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"fail-sgjqb","reconcileID":"383b58a4-92e2-4c7c-9a93-8832ec607889","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"fail-sgjqb\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:20:05.521Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"fail-sgjqb","reconcileID":"3f762f85-198c-4024-a2ef-1a272d6883cd","namespacedName":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:20:05.521Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"fail-sgjqb","reconcileID":"3f762f85-198c-4024-a2ef-1a272d6883cd","mode":"IntegratedOAuth","cluster":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:20:05.521Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"fail-sgjqb","reconcileID":"3f762f85-198c-4024-a2ef-1a272d6883cd","cluster":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:20:05.559Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"391d04bf-c5fb-4093-89b4-039586d546ad"}
{"level":"info","ts":"2026-11-03T15:20:05.559Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"391d04bf-c5fb-4093-89b4-039586d546ad"}
{"level":"info","ts":"2026-11-03T15:20:05.559Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"391d04bf-c5fb-4093-89b4-039586d546ad","1 head service found":"fail-sgjqb-head-svc"}
{"level":"info","ts":"2026-11-03T15:20:05.559Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"391d04bf-c5fb-4093-89b4-039586d546ad"}
{"level":"info","ts":"2026-11-03T15:20:05.584Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"fail-sgjqb","reconcileID":"3f762f85-198c-4024-a2ef-1a272d6883cd","cluster":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:20:05.584Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"ed6f5a1c-c753-43c4-9a33-296eecfd798b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:20:05.584Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"ed6f5a1c-c753-43c4-9a33-296eecfd798b","name":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:20:05.585Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"ed6f5a1c-c753-43c4-9a33-296eecfd798b","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:20:05.585Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"ed6f5a1c-c753-43c4-9a33-296eecfd798b"}
{"level":"info","ts":"2026-11-03T15:20:05.585Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"ed6f5a1c-c753-43c4-9a33-296eecfd798b","RayCluster":"fail-sgjqb","State":""}
{"level":"info","ts":"2026-11-03T15:20:05.585Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"ed6f5a1c-c753-43c4-9a33-296eecfd798b","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:20:05.585Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"ed6f5a1c-c753-43c4-9a33-296eecfd798b","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:20:05.610Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"391d04bf-c5fb-4093-89b4-039586d546ad","error":"certificate ray-head-cert-fail-sgjqb not found: certificates.cert-manager.io \"ray-head-cert-fail-sgjqb\" not found"}
{"level":"error","ts":"2026-11-03T15:20:05.610Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"391d04bf-c5fb-4093-89b4-039586d546ad","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-fail-sgjqb not found: certificates.cert-manager.io \"ray-head-cert-fail-sgjqb\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:20:05.610Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"391d04bf-c5fb-4093-89b4-039586d546ad","error":"mTLS secrets not ready: certificate ray-head-cert-fail-sgjqb not found: certificates.cert-manager.io \"ray-head-cert-fail-sgjqb\" not found"}
{"level":"info","ts":"2026-11-03T15:20:05.659Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"391d04bf-c5fb-4093-89b4-039586d546ad","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"fail-sgjqb","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:20:05.659Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"391d04bf-c5fb-4093-89b4-039586d546ad","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"fail-sgjqb","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:20:05.659Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"391d04bf-c5fb-4093-89b4-039586d546ad","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"fail-sgjqb","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:20:05.659Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"391d04bf-c5fb-4093-89b4-039586d546ad"}
{"level":"error","ts":"2026-11-03T15:20:05.659Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"391d04bf-c5fb-4093-89b4-039586d546ad","error":"mTLS secrets not ready: certificate ray-head-cert-fail-sgjqb not found: certificates.cert-manager.io \"ray-head-cert-fail-sgjqb\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:20:05.660Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"8fd5977f-72b0-4d23-bf78-04b287af7ae9","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:20:05.660Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"8fd5977f-72b0-4d23-bf78-04b287af7ae9","name":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:20:05.660Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"8fd5977f-72b0-4d23-bf78-04b287af7ae9","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:20:05.660Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"8fd5977f-72b0-4d23-bf78-04b287af7ae9"}
{"level":"info","ts":"2026-11-03T15:20:05.660Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"8fd5977f-72b0-4d23-bf78-04b287af7ae9","RayCluster":"fail-sgjqb","State":""}
{"level":"info","ts":"2026-11-03T15:20:05.660Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"8fd5977f-72b0-4d23-bf78-04b287af7ae9","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:20:05.739Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"94abe972-b6f2-4d6c-83a0-628c628bc0d6"}
{"level":"info","ts":"2026-11-03T15:20:05.739Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"94abe972-b6f2-4d6c-83a0-628c628bc0d6"}
{"level":"info","ts":"2026-11-03T15:20:05.740Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"94abe972-b6f2-4d6c-83a0-628c628bc0d6","1 head service found":"fail-sgjqb-head-svc"}
{"level":"info","ts":"2026-11-03T15:20:05.740Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"94abe972-b6f2-4d6c-83a0-628c628bc0d6"}
{"level":"info","ts":"2026-11-03T15:20:05.791Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"94abe972-b6f2-4d6c-83a0-628c628bc0d6","error":"certificate ray-head-cert-fail-sgjqb not found: certificates.cert-manager.io \"ray-head-cert-fail-sgjqb\" not found"}
{"level":"error","ts":"2026-11-03T15:20:05.791Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"94abe972-b6f2-4d6c-83a0-628c628bc0d6","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-fail-sgjqb not found: certificates.cert-manager.io \"ray-head-cert-fail-sgjqb\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:20:05.791Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"94abe972-b6f2-4d6c-83a0-628c628bc0d6","error":"mTLS secrets not ready: certificate ray-head-cert-fail-sgjqb not found: certificates.cert-manager.io \"ray-head-cert-fail-sgjqb\" not found"}
{"level":"info","ts":"2026-11-03T15:20:05.791Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"94abe972-b6f2-4d6c-83a0-628c628bc0d6","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"fail-sgjqb","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:20:05.791Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"94abe972-b6f2-4d6c-83a0-628c628bc0d6","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"fail-sgjqb","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:20:05.791Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"94abe972-b6f2-4d6c-83a0-628c628bc0d6","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"fail-sgjqb","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:20:05.791Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"94abe972-b6f2-4d6c-83a0-628c628bc0d6"}
{"level":"error","ts":"2026-11-03T15:20:05.791Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"94abe972-b6f2-4d6c-83a0-628c628bc0d6","error":"mTLS secrets not ready: certificate ray-head-cert-fail-sgjqb not found: certificates.cert-manager.io \"ray-head-cert-fail-sgjqb\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:20:05.952Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"2254d5ab-37a5-487d-b5d4-a46509515103"}
{"level":"info","ts":"2026-11-03T15:20:05.952Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"2254d5ab-37a5-487d-b5d4-a46509515103"}
{"level":"info","ts":"2026-11-03T15:20:05.952Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"2254d5ab-37a5-487d-b5d4-a46509515103","1 head service found":"fail-sgjqb-head-svc"}
{"level":"info","ts":"2026-11-03T15:20:05.952Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"2254d5ab-37a5-487d-b5d4-a46509515103"}
{"level":"info","ts":"2026-11-03T15:20:06.004Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"2254d5ab-37a5-487d-b5d4-a46509515103","error":"certificate ray-head-cert-fail-sgjqb is not ready at current generation 1"}
{"level":"error","ts":"2026-11-03T15:20:06.004Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"2254d5ab-37a5-487d-b5d4-a46509515103","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-fail-sgjqb is not ready at current generation 1","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:20:06.004Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"2254d5ab-37a5-487d-b5d4-a46509515103","error":"mTLS secrets not ready: certificate ray-head-cert-fail-sgjqb is not ready at current generation 1"}
{"level":"info","ts":"2026-11-03T15:20:06.004Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"2254d5ab-37a5-487d-b5d4-a46509515103","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"fail-sgjqb","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:20:06.004Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"2254d5ab-37a5-487d-b5d4-a46509515103","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"fail-sgjqb","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:20:06.004Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"2254d5ab-37a5-487d-b5d4-a46509515103","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"fail-sgjqb","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:20:06.004Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"2254d5ab-37a5-487d-b5d4-a46509515103"}
{"level":"error","ts":"2026-11-03T15:20:06.004Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"2254d5ab-37a5-487d-b5d4-a46509515103","error":"mTLS secrets not ready: certificate ray-head-cert-fail-sgjqb is not ready at current generation 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:20:06.153Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"354375ef-d0bd-4667-9b64-1274cab8ed77","certificate":"ray-head-cert-fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:20:06.153Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"354375ef-d0bd-4667-9b64-1274cab8ed77"}
{"level":"info","ts":"2026-11-03T15:20:06.153Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"62400381-1854-4c7d-bf10-2168c5b8533b","cluster":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:20:06.325Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"2424867c-e51f-4f82-a55f-524a477f27b2"}
{"level":"info","ts":"2026-11-03T15:20:06.325Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"2424867c-e51f-4f82-a55f-524a477f27b2"}
{"level":"info","ts":"2026-11-03T15:20:06.325Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"2424867c-e51f-4f82-a55f-524a477f27b2","1 head service found":"fail-sgjqb-head-svc"}
{"level":"info","ts":"2026-11-03T15:20:06.325Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"2424867c-e51f-4f82-a55f-524a477f27b2"}
{"level":"info","ts":"2026-11-03T15:20:06.378Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"2424867c-e51f-4f82-a55f-524a477f27b2","error":"certificate ray-head-cert-fail-sgjqb is not ready at current generation 1"}
{"level":"error","ts":"2026-11-03T15:20:06.378Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"2424867c-e51f-4f82-a55f-524a477f27b2","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-fail-sgjqb is not ready at current generation 1","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:20:06.378Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"2424867c-e51f-4f82-a55f-524a477f27b2","error":"mTLS secrets not ready: certificate ray-head-cert-fail-sgjqb is not ready at current generation 1"}
{"level":"info","ts":"2026-11-03T15:20:06.378Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"2424867c-e51f-4f82-a55f-524a477f27b2","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"fail-sgjqb","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:20:06.378Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"2424867c-e51f-4f82-a55f-524a477f27b2","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"fail-sgjqb","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:20:06.378Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"2424867c-e51f-4f82-a55f-524a477f27b2","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"fail-sgjqb","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:20:06.378Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"2424867c-e51f-4f82-a55f-524a477f27b2"}
{"level":"error","ts":"2026-11-03T15:20:06.378Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"2424867c-e51f-4f82-a55f-524a477f27b2","error":"mTLS secrets not ready: certificate ray-head-cert-fail-sgjqb is not ready at current generation 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:20:06.621Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"62400381-1854-4c7d-bf10-2168c5b8533b","certificate":"ray-head-cert-fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:20:06.621Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"62400381-1854-4c7d-bf10-2168c5b8533b"}
{"level":"info","ts":"2026-11-03T15:20:07.019Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"d25c25b9-5b29-4ccd-bdea-a9df067420a4"}
{"level":"info","ts":"2026-11-03T15:20:07.019Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"d25c25b9-5b29-4ccd-bdea-a9df067420a4"}
{"level":"info","ts":"2026-11-03T15:20:07.019Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"d25c25b9-5b29-4ccd-bdea-a9df067420a4","1 head service found":"fail-sgjqb-head-svc"}
{"level":"info","ts":"2026-11-03T15:20:07.019Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"d25c25b9-5b29-4ccd-bdea-a9df067420a4"}
{"level":"info","ts":"2026-11-03T15:20:07.071Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"d25c25b9-5b29-4ccd-bdea-a9df067420a4","error":"certificate ray-head-cert-fail-sgjqb is not ready at current generation 1"}
{"level":"error","ts":"2026-11-03T15:20:07.071Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"d25c25b9-5b29-4ccd-bdea-a9df067420a4","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-fail-sgjqb is not ready at current generation 1","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:20:07.071Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"d25c25b9-5b29-4ccd-bdea-a9df067420a4","error":"mTLS secrets not ready: certificate ray-head-cert-fail-sgjqb is not ready at current generation 1"}
{"level":"info","ts":"2026-11-03T15:20:07.071Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"d25c25b9-5b29-4ccd-bdea-a9df067420a4","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"fail-sgjqb","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:20:07.071Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"d25c25b9-5b29-4ccd-bdea-a9df067420a4","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"fail-sgjqb","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:20:07.071Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"d25c25b9-5b29-4ccd-bdea-a9df067420a4","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"fail-sgjqb","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:20:07.071Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"d25c25b9-5b29-4ccd-bdea-a9df067420a4"}
{"level":"error","ts":"2026-11-03T15:20:07.071Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"d25c25b9-5b29-4ccd-bdea-a9df067420a4","error":"mTLS secrets not ready: certificate ray-head-cert-fail-sgjqb is not ready at current generation 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:20:08.353Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"1fdf43fe-92af-4f68-9e40-09f35ed0e60e"}
{"level":"info","ts":"2026-11-03T15:20:08.353Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"1fdf43fe-92af-4f68-9e40-09f35ed0e60e"}
{"level":"info","ts":"2026-11-03T15:20:08.353Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"1fdf43fe-92af-4f68-9e40-09f35ed0e60e","1 head service found":"fail-sgjqb-head-svc"}
{"level":"info","ts":"2026-11-03T15:20:08.353Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"1fdf43fe-92af-4f68-9e40-09f35ed0e60e"}
{"level":"info","ts":"2026-11-03T15:20:08.454Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"1fdf43fe-92af-4f68-9e40-09f35ed0e60e","mode":"IntegratedOAuth","cluster":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:20:08.454Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"1fdf43fe-92af-4f68-9e40-09f35ed0e60e","cluster":"fail-sgjqb","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-11-03T15:20:08.454Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"1fdf43fe-92af-4f68-9e40-09f35ed0e60e","cluster":"fail-sgjqb","authType":"IntegratedOAuth","serviceAccount":"fail-sgjqb-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-11-03T15:20:08.454Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"1fdf43fe-92af-4f68-9e40-09f35ed0e60e","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"fail-sgjqb","ray.io/group":"headgroup","ray.io/identifier":"fail-sgjqb-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-11-03T15:20:08.454Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"1fdf43fe-92af-4f68-9e40-09f35ed0e60e","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-11-03T15:20:08.454Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"1fdf43fe-92af-4f68-9e40-09f35ed0e60e","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-11-03T15:20:08.454Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"1fdf43fe-92af-4f68-9e40-09f35ed0e60e","rayNodeType":"head","generatedCmd":"ulimit -n ${RAY_START_ULIMIT_OPEN_FILES:-65536}; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-11-03T15:20:08.454Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"1fdf43fe-92af-4f68-9e40-09f35ed0e60e","enabled":true}
{"level":"info","ts":"2026-11-03T15:20:08.567Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"1fdf43fe-92af-4f68-9e40-09f35ed0e60e","name":"fail-sgjqb-head-8pdmr"}
{"level":"info","ts":"2026-11-03T15:20:08.567Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"1fdf43fe-92af-4f68-9e40-09f35ed0e60e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:20:08.567Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"1fdf43fe-92af-4f68-9e40-09f35ed0e60e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:20:08.567Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"1fdf43fe-92af-4f68-9e40-09f35ed0e60e","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-11-03T15:20:08.567Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"1fdf43fe-92af-4f68-9e40-09f35ed0e60e","group":"small-group","indices":{}}
{"level":"info","ts":"2026-11-03T15:20:08.567Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"1fdf43fe-92af-4f68-9e40-09f35ed0e60e","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-11-03T15:20:08.661Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"5b41aa61-2d29-46a4-a5dd-232169ba3288","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:20:08.661Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"5b41aa61-2d29-46a4-a5dd-232169ba3288","name":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:20:08.661Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"5b41aa61-2d29-46a4-a5dd-232169ba3288","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:20:08.661Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"5b41aa61-2d29-46a4-a5dd-232169ba3288"}
{"level":"info","ts":"2026-11-03T15:20:08.661Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"5b41aa61-2d29-46a4-a5dd-232169ba3288","RayCluster":"fail-sgjqb","State":""}
{"level":"info","ts":"2026-11-03T15:20:08.661Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"5b41aa61-2d29-46a4-a5dd-232169ba3288","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:20:08.668Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"1fdf43fe-92af-4f68-9e40-09f35ed0e60e","creating worker for group":"small-group","index":0,"total":1,"replicaIndex":0}
{"level":"info","ts":"2026-11-03T15:20:08.668Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"1fdf43fe-92af-4f68-9e40-09f35ed0e60e","nodeType":"worker","rayStartParams":{"address":"fail-sgjqb-head-svc.test-ns-kbx4w.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-11-03T15:20:08.668Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"1fdf43fe-92af-4f68-9e40-09f35ed0e60e","rayStartCmd":"ray start  --address=fail-sgjqb-head-svc.test-ns-kbx4w.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-11-03T15:20:08.668Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"1fdf43fe-92af-4f68-9e40-09f35ed0e60e","rayNodeType":"worker","generatedCmd":"ulimit -n ${RAY_START_ULIMIT_OPEN_FILES:-65536}; ray start  --address=fail-sgjqb-head-svc.test-ns-kbx4w.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-11-03T15:20:08.668Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"1fdf43fe-92af-4f68-9e40-09f35ed0e60e","enabled":true}
{"level":"info","ts":"2026-11-03T15:20:08.739Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"1fdf43fe-92af-4f68-9e40-09f35ed0e60e","name":"fail-sgjqb-small-group-worker-k4f2l"}
{"level":"info","ts":"2026-11-03T15:20:08.739Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"1fdf43fe-92af-4f68-9e40-09f35ed0e60e","podName":"fail-sgjqb-head-8pdmr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:20:08.739Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"1fdf43fe-92af-4f68-9e40-09f35ed0e60e","podName":"fail-sgjqb-head-8pdmr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:20:08.739Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"1fdf43fe-92af-4f68-9e40-09f35ed0e60e","name":"fail-sgjqb","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:08Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:20:08.800Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"119322fb-b90f-42fe-be4c-8d3806097b98"}
{"level":"info","ts":"2026-11-03T15:20:08.800Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"119322fb-b90f-42fe-be4c-8d3806097b98"}
{"level":"info","ts":"2026-11-03T15:20:08.800Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"119322fb-b90f-42fe-be4c-8d3806097b98","1 head service found":"fail-sgjqb-head-svc"}
{"level":"info","ts":"2026-11-03T15:20:08.801Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"119322fb-b90f-42fe-be4c-8d3806097b98","Found 1 head Pod":"fail-sgjqb-head-8pdmr","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:20:08.801Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"119322fb-b90f-42fe-be4c-8d3806097b98","head Pod":"fail-sgjqb-head-8pdmr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-sgjqb-head-8pdmr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:20:08.801Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"119322fb-b90f-42fe-be4c-8d3806097b98","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:20:08.801Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"119322fb-b90f-42fe-be4c-8d3806097b98","worker Pod":"fail-sgjqb-small-group-worker-k4f2l","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-sgjqb-small-group-worker-k4f2l. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:20:08.801Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"119322fb-b90f-42fe-be4c-8d3806097b98","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:20:08.801Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"119322fb-b90f-42fe-be4c-8d3806097b98","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:20:08.801Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"8149e970-caa0-4484-8cb8-e5777b395e2b","cluster":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:20:08.801Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"119322fb-b90f-42fe-be4c-8d3806097b98","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:20:08.801Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"119322fb-b90f-42fe-be4c-8d3806097b98","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:20:08.801Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"119322fb-b90f-42fe-be4c-8d3806097b98","podName":"fail-sgjqb-head-8pdmr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:20:08.801Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"119322fb-b90f-42fe-be4c-8d3806097b98","podName":"fail-sgjqb-head-8pdmr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:20:08.801Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"119322fb-b90f-42fe-be4c-8d3806097b98","name":"fail-sgjqb","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:08Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:20:08.801Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"335ad63f-e96f-49ca-8be2-622bd4ec0721","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:20:08.801Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"335ad63f-e96f-49ca-8be2-622bd4ec0721","name":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:20:08.801Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"335ad63f-e96f-49ca-8be2-622bd4ec0721","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:20:08.801Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"335ad63f-e96f-49ca-8be2-622bd4ec0721"}
{"level":"info","ts":"2026-11-03T15:20:08.801Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"335ad63f-e96f-49ca-8be2-622bd4ec0721","RayCluster":"fail-sgjqb","State":""}
{"level":"info","ts":"2026-11-03T15:20:08.801Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"335ad63f-e96f-49ca-8be2-622bd4ec0721","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:08Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:20:08.801Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"335ad63f-e96f-49ca-8be2-622bd4ec0721","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:20:08.856Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"119322fb-b90f-42fe-be4c-8d3806097b98","name":"fail-sgjqb","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-sgjqb\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-kbx4w","name":"fail-sgjqb"}}
{"level":"info","ts":"2026-11-03T15:20:08.856Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"119322fb-b90f-42fe-be4c-8d3806097b98"}
{"level":"error","ts":"2026-11-03T15:20:08.856Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"119322fb-b90f-42fe-be4c-8d3806097b98","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-sgjqb\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:20:08.856Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"6f95b861-c718-4536-ba71-4ecc0d6cf291"}
{"level":"info","ts":"2026-11-03T15:20:08.856Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"6f95b861-c718-4536-ba71-4ecc0d6cf291"}
{"level":"info","ts":"2026-11-03T15:20:08.856Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"6f95b861-c718-4536-ba71-4ecc0d6cf291","1 head service found":"fail-sgjqb-head-svc"}
{"level":"info","ts":"2026-11-03T15:20:08.856Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"6f95b861-c718-4536-ba71-4ecc0d6cf291","Found 1 head Pod":"fail-sgjqb-head-8pdmr","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:20:08.856Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"6f95b861-c718-4536-ba71-4ecc0d6cf291","head Pod":"fail-sgjqb-head-8pdmr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-sgjqb-head-8pdmr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:20:08.856Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"6f95b861-c718-4536-ba71-4ecc0d6cf291","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:20:08.856Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"6f95b861-c718-4536-ba71-4ecc0d6cf291","worker Pod":"fail-sgjqb-small-group-worker-k4f2l","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-sgjqb-small-group-worker-k4f2l. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:20:08.856Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"6f95b861-c718-4536-ba71-4ecc0d6cf291","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:20:08.856Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"6f95b861-c718-4536-ba71-4ecc0d6cf291","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:20:08.856Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"6f95b861-c718-4536-ba71-4ecc0d6cf291","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:20:08.857Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"6f95b861-c718-4536-ba71-4ecc0d6cf291","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:20:08.857Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"6f95b861-c718-4536-ba71-4ecc0d6cf291","podName":"fail-sgjqb-head-8pdmr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:20:08.857Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"6f95b861-c718-4536-ba71-4ecc0d6cf291","podName":"fail-sgjqb-head-8pdmr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:20:08.857Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"6f95b861-c718-4536-ba71-4ecc0d6cf291","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:20:08.857Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"6f95b861-c718-4536-ba71-4ecc0d6cf291","seconds":300}
{"level":"info","ts":"2026-11-03T15:20:08.874Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"03d92d79-f248-4c0e-bcf4-25a21490dd48"}
{"level":"info","ts":"2026-11-03T15:20:08.874Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"03d92d79-f248-4c0e-bcf4-25a21490dd48"}
{"level":"info","ts":"2026-11-03T15:20:08.874Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"03d92d79-f248-4c0e-bcf4-25a21490dd48","1 head service found":"fail-sgjqb-head-svc"}
{"level":"info","ts":"2026-11-03T15:20:08.874Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"03d92d79-f248-4c0e-bcf4-25a21490dd48","Found 1 head Pod":"fail-sgjqb-head-8pdmr","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:20:08.874Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"03d92d79-f248-4c0e-bcf4-25a21490dd48","head Pod":"fail-sgjqb-head-8pdmr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-sgjqb-head-8pdmr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:20:08.874Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"03d92d79-f248-4c0e-bcf4-25a21490dd48","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:20:08.874Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"03d92d79-f248-4c0e-bcf4-25a21490dd48","worker Pod":"fail-sgjqb-small-group-worker-k4f2l","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-sgjqb-small-group-worker-k4f2l. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:20:08.874Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"03d92d79-f248-4c0e-bcf4-25a21490dd48","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:20:08.874Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"03d92d79-f248-4c0e-bcf4-25a21490dd48","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:20:08.874Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"03d92d79-f248-4c0e-bcf4-25a21490dd48","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:20:08.874Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"03d92d79-f248-4c0e-bcf4-25a21490dd48","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:20:08.874Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"03d92d79-f248-4c0e-bcf4-25a21490dd48","podName":"fail-sgjqb-head-8pdmr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:20:08.875Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"03d92d79-f248-4c0e-bcf4-25a21490dd48","podName":"fail-sgjqb-head-8pdmr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:20:08.875Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"03d92d79-f248-4c0e-bcf4-25a21490dd48","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:20:08.875Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"03d92d79-f248-4c0e-bcf4-25a21490dd48","seconds":300}
{"level":"info","ts":"2026-11-03T15:20:08.908Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"dec3058d-2cd7-47f7-ad0d-b9cc92949151","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:20:08.908Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"dec3058d-2cd7-47f7-ad0d-b9cc92949151","name":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:20:08.908Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"dec3058d-2cd7-47f7-ad0d-b9cc92949151","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:20:08.908Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"dec3058d-2cd7-47f7-ad0d-b9cc92949151"}
{"level":"info","ts":"2026-11-03T15:20:08.908Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"dec3058d-2cd7-47f7-ad0d-b9cc92949151","RayCluster":"fail-sgjqb","State":""}
{"level":"info","ts":"2026-11-03T15:20:08.908Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"dec3058d-2cd7-47f7-ad0d-b9cc92949151","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:08Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:08Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:20:09.484Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"0d6e1725-6224-4e35-8897-5b15ae3d6ef2"}
{"level":"info","ts":"2026-11-03T15:20:09.484Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"0d6e1725-6224-4e35-8897-5b15ae3d6ef2"}
{"level":"info","ts":"2026-11-03T15:20:09.484Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"0d6e1725-6224-4e35-8897-5b15ae3d6ef2","1 head service found":"fail-sgjqb-head-svc"}
{"level":"info","ts":"2026-11-03T15:20:09.485Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"0d6e1725-6224-4e35-8897-5b15ae3d6ef2","Found 1 head Pod":"fail-sgjqb-head-8pdmr","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:20:09.485Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"0d6e1725-6224-4e35-8897-5b15ae3d6ef2","head Pod":"fail-sgjqb-head-8pdmr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-sgjqb-head-8pdmr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:20:09.485Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"0d6e1725-6224-4e35-8897-5b15ae3d6ef2","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:20:09.485Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"0d6e1725-6224-4e35-8897-5b15ae3d6ef2","worker Pod":"fail-sgjqb-small-group-worker-k4f2l","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-sgjqb-small-group-worker-k4f2l. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:20:09.485Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"0d6e1725-6224-4e35-8897-5b15ae3d6ef2","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:20:09.485Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"0d6e1725-6224-4e35-8897-5b15ae3d6ef2","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:20:09.485Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"0d6e1725-6224-4e35-8897-5b15ae3d6ef2","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:20:09.485Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"0d6e1725-6224-4e35-8897-5b15ae3d6ef2","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:20:09.485Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"0d6e1725-6224-4e35-8897-5b15ae3d6ef2","podName":"fail-sgjqb-head-8pdmr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:20:09.485Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"0d6e1725-6224-4e35-8897-5b15ae3d6ef2","podName":"fail-sgjqb-head-8pdmr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:20:09.485Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"0d6e1725-6224-4e35-8897-5b15ae3d6ef2","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:20:09.485Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"0d6e1725-6224-4e35-8897-5b15ae3d6ef2","seconds":300}
{"level":"info","ts":"2026-11-03T15:20:09.619Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"c0e4475d-6ac1-4a48-811b-17a133fc9bf1"}
{"level":"info","ts":"2026-11-03T15:20:09.619Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"c0e4475d-6ac1-4a48-811b-17a133fc9bf1"}
{"level":"info","ts":"2026-11-03T15:20:09.619Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"c0e4475d-6ac1-4a48-811b-17a133fc9bf1","1 head service found":"fail-sgjqb-head-svc"}
{"level":"info","ts":"2026-11-03T15:20:09.619Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"c0e4475d-6ac1-4a48-811b-17a133fc9bf1","Found 1 head Pod":"fail-sgjqb-head-8pdmr","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:20:09.619Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"c0e4475d-6ac1-4a48-811b-17a133fc9bf1","head Pod":"fail-sgjqb-head-8pdmr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-sgjqb-head-8pdmr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:20:09.619Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"c0e4475d-6ac1-4a48-811b-17a133fc9bf1","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:20:09.619Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"c0e4475d-6ac1-4a48-811b-17a133fc9bf1","worker Pod":"fail-sgjqb-small-group-worker-k4f2l","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-sgjqb-small-group-worker-k4f2l. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:20:09.619Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"c0e4475d-6ac1-4a48-811b-17a133fc9bf1","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:20:09.619Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"c0e4475d-6ac1-4a48-811b-17a133fc9bf1","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:20:09.619Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"c0e4475d-6ac1-4a48-811b-17a133fc9bf1","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:20:09.619Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"c0e4475d-6ac1-4a48-811b-17a133fc9bf1","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:20:09.619Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"c0e4475d-6ac1-4a48-811b-17a133fc9bf1","podName":"fail-sgjqb-head-8pdmr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:20:09.619Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"c0e4475d-6ac1-4a48-811b-17a133fc9bf1","podName":"fail-sgjqb-head-8pdmr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:20:09.620Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"c0e4475d-6ac1-4a48-811b-17a133fc9bf1","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:20:09.620Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"c0e4475d-6ac1-4a48-811b-17a133fc9bf1","seconds":300}
{"level":"info","ts":"2026-11-03T15:20:10.098Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"2db467e1-de8a-45e5-b8b7-b52b269184b0"}
{"level":"info","ts":"2026-11-03T15:20:10.098Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"2db467e1-de8a-45e5-b8b7-b52b269184b0"}
{"level":"info","ts":"2026-11-03T15:20:10.098Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"2db467e1-de8a-45e5-b8b7-b52b269184b0","1 head service found":"fail-sgjqb-head-svc"}
{"level":"info","ts":"2026-11-03T15:20:10.098Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"2db467e1-de8a-45e5-b8b7-b52b269184b0","Found 1 head Pod":"fail-sgjqb-head-8pdmr","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:20:10.098Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"2db467e1-de8a-45e5-b8b7-b52b269184b0","head Pod":"fail-sgjqb-head-8pdmr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-sgjqb-head-8pdmr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:20:10.098Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"2db467e1-de8a-45e5-b8b7-b52b269184b0","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:20:10.098Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"2db467e1-de8a-45e5-b8b7-b52b269184b0","worker Pod":"fail-sgjqb-small-group-worker-k4f2l","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-sgjqb-small-group-worker-k4f2l. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:20:10.098Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"2db467e1-de8a-45e5-b8b7-b52b269184b0","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:20:10.098Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"2db467e1-de8a-45e5-b8b7-b52b269184b0","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:20:10.098Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"2db467e1-de8a-45e5-b8b7-b52b269184b0","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:20:10.098Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"2db467e1-de8a-45e5-b8b7-b52b269184b0","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:20:10.098Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"2db467e1-de8a-45e5-b8b7-b52b269184b0","podName":"fail-sgjqb-head-8pdmr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:20:10.099Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"2db467e1-de8a-45e5-b8b7-b52b269184b0","podName":"fail-sgjqb-head-8pdmr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:20:10.099Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"2db467e1-de8a-45e5-b8b7-b52b269184b0","name":"fail-sgjqb","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:08Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:20:10.160Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"5ddc864e-d760-465a-8cdb-ba19bfa30a56"}
{"level":"info","ts":"2026-11-03T15:20:10.161Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"5ddc864e-d760-465a-8cdb-ba19bfa30a56"}
{"level":"info","ts":"2026-11-03T15:20:10.161Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"5ddc864e-d760-465a-8cdb-ba19bfa30a56","1 head service found":"fail-sgjqb-head-svc"}
{"level":"info","ts":"2026-11-03T15:20:10.161Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"5ddc864e-d760-465a-8cdb-ba19bfa30a56","Found 1 head Pod":"fail-sgjqb-head-8pdmr","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:20:10.161Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"5ddc864e-d760-465a-8cdb-ba19bfa30a56","head Pod":"fail-sgjqb-head-8pdmr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-sgjqb-head-8pdmr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:20:10.161Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"5ddc864e-d760-465a-8cdb-ba19bfa30a56","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:20:10.161Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"5ddc864e-d760-465a-8cdb-ba19bfa30a56","worker Pod":"fail-sgjqb-small-group-worker-k4f2l","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-sgjqb-small-group-worker-k4f2l. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:20:10.161Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"5ddc864e-d760-465a-8cdb-ba19bfa30a56","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:20:10.161Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"5ddc864e-d760-465a-8cdb-ba19bfa30a56","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:20:10.161Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"5ddc864e-d760-465a-8cdb-ba19bfa30a56","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:20:10.161Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"5ddc864e-d760-465a-8cdb-ba19bfa30a56","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:20:10.161Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"5ddc864e-d760-465a-8cdb-ba19bfa30a56","podName":"fail-sgjqb-head-8pdmr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:20:10.161Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"5ddc864e-d760-465a-8cdb-ba19bfa30a56","podName":"fail-sgjqb-head-8pdmr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:20:10.161Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"89f8d7e7-b584-43b3-95bd-df53c89f6094","cluster":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:20:10.161Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"5ddc864e-d760-465a-8cdb-ba19bfa30a56","name":"fail-sgjqb","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:08Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:20:10.161Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"8efc9080-4f0c-492b-af87-e1a660363d69","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:20:10.161Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"8efc9080-4f0c-492b-af87-e1a660363d69","name":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:20:10.161Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"8efc9080-4f0c-492b-af87-e1a660363d69","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:20:10.161Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"8efc9080-4f0c-492b-af87-e1a660363d69"}
{"level":"info","ts":"2026-11-03T15:20:10.162Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"8efc9080-4f0c-492b-af87-e1a660363d69","RayCluster":"fail-sgjqb","State":""}
{"level":"info","ts":"2026-11-03T15:20:10.162Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"8efc9080-4f0c-492b-af87-e1a660363d69","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:08Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:20:10.162Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"8efc9080-4f0c-492b-af87-e1a660363d69","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:20:10.216Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"5ddc864e-d760-465a-8cdb-ba19bfa30a56","name":"fail-sgjqb","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-sgjqb\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-kbx4w","name":"fail-sgjqb"}}
{"level":"info","ts":"2026-11-03T15:20:10.216Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"5ddc864e-d760-465a-8cdb-ba19bfa30a56"}
{"level":"error","ts":"2026-11-03T15:20:10.216Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"5ddc864e-d760-465a-8cdb-ba19bfa30a56","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-sgjqb\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:20:10.221Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"5b674abc-63b6-4744-8021-bfc47a8ddec6"}
{"level":"info","ts":"2026-11-03T15:20:10.221Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"5b674abc-63b6-4744-8021-bfc47a8ddec6"}
{"level":"info","ts":"2026-11-03T15:20:10.221Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"5b674abc-63b6-4744-8021-bfc47a8ddec6","1 head service found":"fail-sgjqb-head-svc"}
{"level":"info","ts":"2026-11-03T15:20:10.221Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"5b674abc-63b6-4744-8021-bfc47a8ddec6","Found 1 head Pod":"fail-sgjqb-head-8pdmr","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:20:10.221Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"5b674abc-63b6-4744-8021-bfc47a8ddec6","head Pod":"fail-sgjqb-head-8pdmr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-sgjqb-head-8pdmr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:20:10.221Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"5b674abc-63b6-4744-8021-bfc47a8ddec6","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:20:10.221Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"5b674abc-63b6-4744-8021-bfc47a8ddec6","worker Pod":"fail-sgjqb-small-group-worker-k4f2l","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-sgjqb-small-group-worker-k4f2l. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:20:10.221Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"5b674abc-63b6-4744-8021-bfc47a8ddec6","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:20:10.221Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"5b674abc-63b6-4744-8021-bfc47a8ddec6","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:20:10.221Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"5b674abc-63b6-4744-8021-bfc47a8ddec6","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:20:10.221Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"5b674abc-63b6-4744-8021-bfc47a8ddec6","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:20:10.222Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"5b674abc-63b6-4744-8021-bfc47a8ddec6","podName":"fail-sgjqb-head-8pdmr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:20:10.222Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"5b674abc-63b6-4744-8021-bfc47a8ddec6","podName":"fail-sgjqb-head-8pdmr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:20:10.222Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"5b674abc-63b6-4744-8021-bfc47a8ddec6","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:20:10.222Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"5b674abc-63b6-4744-8021-bfc47a8ddec6","seconds":300}
{"level":"info","ts":"2026-11-03T15:20:10.223Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"57d5981b-0d0a-477d-ac15-05c1965582c7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:20:10.223Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"57d5981b-0d0a-477d-ac15-05c1965582c7","name":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:20:10.223Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"57d5981b-0d0a-477d-ac15-05c1965582c7","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:20:10.223Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"57d5981b-0d0a-477d-ac15-05c1965582c7"}
{"level":"info","ts":"2026-11-03T15:20:10.223Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"57d5981b-0d0a-477d-ac15-05c1965582c7","RayCluster":"fail-sgjqb","State":""}
{"level":"info","ts":"2026-11-03T15:20:10.223Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"57d5981b-0d0a-477d-ac15-05c1965582c7","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:20:10.467Z","logger":"controllers.RayClusterMTLS","msg":"Updating head certificate","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"89f8d7e7-b584-43b3-95bd-df53c89f6094","certificate":"ray-head-cert-fail-sgjqb","ipAddresses":["10.132.0.39","127.0.0.1"]}
{"level":"info","ts":"2026-11-03T15:20:10.724Z","logger":"controllers.RayClusterMTLS","msg":"Updating worker certificate","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"89f8d7e7-b584-43b3-95bd-df53c89f6094","certificate":"ray-worker-cert-fail-sgjqb","ipAddresses":["10.134.0.39","127.0.0.1"]}
{"level":"info","ts":"2026-11-03T15:20:10.980Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"89f8d7e7-b584-43b3-95bd-df53c89f6094","certificate":"ray-head-cert-fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:20:10.980Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"89f8d7e7-b584-43b3-95bd-df53c89f6094"}
{"level":"info","ts":"2026-11-03T15:20:13.224Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"06271339-63ac-40e7-91c7-e988e19ec7a3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:20:13.224Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"06271339-63ac-40e7-91c7-e988e19ec7a3","name":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:20:13.224Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"06271339-63ac-40e7-91c7-e988e19ec7a3","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:20:13.224Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"06271339-63ac-40e7-91c7-e988e19ec7a3"}
{"level":"info","ts":"2026-11-03T15:20:13.224Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"06271339-63ac-40e7-91c7-e988e19ec7a3","RayCluster":"fail-sgjqb","State":""}
{"level":"info","ts":"2026-11-03T15:20:13.224Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"06271339-63ac-40e7-91c7-e988e19ec7a3","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:20:16.224Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"d5d73a82-3494-4bde-b921-cc9ff77607ac","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:20:16.224Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"d5d73a82-3494-4bde-b921-cc9ff77607ac","name":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:20:16.224Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"d5d73a82-3494-4bde-b921-cc9ff77607ac","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:20:16.225Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"d5d73a82-3494-4bde-b921-cc9ff77607ac"}
{"level":"info","ts":"2026-11-03T15:20:16.225Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"d5d73a82-3494-4bde-b921-cc9ff77607ac","RayCluster":"fail-sgjqb","State":""}
{"level":"info","ts":"2026-11-03T15:20:16.225Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"d5d73a82-3494-4bde-b921-cc9ff77607ac","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:20:19.225Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"d9ab1a1a-a8ea-404b-8969-3f1574255807","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:20:19.225Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"d9ab1a1a-a8ea-404b-8969-3f1574255807","name":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:20:19.225Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"d9ab1a1a-a8ea-404b-8969-3f1574255807","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:20:19.225Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"d9ab1a1a-a8ea-404b-8969-3f1574255807"}
{"level":"info","ts":"2026-11-03T15:20:19.225Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"d9ab1a1a-a8ea-404b-8969-3f1574255807","RayCluster":"fail-sgjqb","State":""}
{"level":"info","ts":"2026-11-03T15:20:19.225Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"d9ab1a1a-a8ea-404b-8969-3f1574255807","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:20:22.226Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"ce6f2e5d-8081-4c12-86e1-c679d0772459","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:20:22.226Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"ce6f2e5d-8081-4c12-86e1-c679d0772459","name":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:20:22.226Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"ce6f2e5d-8081-4c12-86e1-c679d0772459","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:20:22.226Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"ce6f2e5d-8081-4c12-86e1-c679d0772459"}
{"level":"info","ts":"2026-11-03T15:20:22.226Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"ce6f2e5d-8081-4c12-86e1-c679d0772459","RayCluster":"fail-sgjqb","State":""}
{"level":"info","ts":"2026-11-03T15:20:22.226Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"ce6f2e5d-8081-4c12-86e1-c679d0772459","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:20:25.226Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"135a1c4b-6657-4789-9717-b5e098f0641c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:20:25.227Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"135a1c4b-6657-4789-9717-b5e098f0641c","name":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:20:25.227Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"135a1c4b-6657-4789-9717-b5e098f0641c","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:20:25.227Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"135a1c4b-6657-4789-9717-b5e098f0641c"}
{"level":"info","ts":"2026-11-03T15:20:25.227Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"135a1c4b-6657-4789-9717-b5e098f0641c","RayCluster":"fail-sgjqb","State":""}
{"level":"info","ts":"2026-11-03T15:20:25.227Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"135a1c4b-6657-4789-9717-b5e098f0641c","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:20:28.227Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"cc49584b-9a04-40b9-8a4a-272147d56393","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:20:28.227Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"cc49584b-9a04-40b9-8a4a-272147d56393","name":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:20:28.227Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"cc49584b-9a04-40b9-8a4a-272147d56393","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:20:28.228Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"cc49584b-9a04-40b9-8a4a-272147d56393"}
{"level":"info","ts":"2026-11-03T15:20:28.228Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"cc49584b-9a04-40b9-8a4a-272147d56393","RayCluster":"fail-sgjqb","State":""}
{"level":"info","ts":"2026-11-03T15:20:28.228Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"cc49584b-9a04-40b9-8a4a-272147d56393","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:20:31.228Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"3372a5fe-5bb9-4d90-87f3-ed845a085c1d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:20:31.228Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"3372a5fe-5bb9-4d90-87f3-ed845a085c1d","name":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:20:31.228Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"3372a5fe-5bb9-4d90-87f3-ed845a085c1d","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:20:31.228Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"3372a5fe-5bb9-4d90-87f3-ed845a085c1d"}
{"level":"info","ts":"2026-11-03T15:20:31.228Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"3372a5fe-5bb9-4d90-87f3-ed845a085c1d","RayCluster":"fail-sgjqb","State":""}
{"level":"info","ts":"2026-11-03T15:20:31.228Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"3372a5fe-5bb9-4d90-87f3-ed845a085c1d","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:20:34.229Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"9444328f-30ca-4f6d-aaeb-85ed415b9b96","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:20:34.229Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"9444328f-30ca-4f6d-aaeb-85ed415b9b96","name":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:20:34.229Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"9444328f-30ca-4f6d-aaeb-85ed415b9b96","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:20:34.229Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"9444328f-30ca-4f6d-aaeb-85ed415b9b96"}
{"level":"info","ts":"2026-11-03T15:20:34.229Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"9444328f-30ca-4f6d-aaeb-85ed415b9b96","RayCluster":"fail-sgjqb","State":""}
{"level":"info","ts":"2026-11-03T15:20:34.229Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"9444328f-30ca-4f6d-aaeb-85ed415b9b96","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:20:37.230Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"a7c0e2b2-51ef-458b-9ec9-73da17905d3f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:20:37.230Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"a7c0e2b2-51ef-458b-9ec9-73da17905d3f","name":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:20:37.230Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"a7c0e2b2-51ef-458b-9ec9-73da17905d3f","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:20:37.231Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"a7c0e2b2-51ef-458b-9ec9-73da17905d3f"}
{"level":"info","ts":"2026-11-03T15:20:37.231Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"a7c0e2b2-51ef-458b-9ec9-73da17905d3f","RayCluster":"fail-sgjqb","State":""}
{"level":"info","ts":"2026-11-03T15:20:37.231Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"a7c0e2b2-51ef-458b-9ec9-73da17905d3f","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:20:40.232Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"d5af6719-eb7a-4618-8828-fc8a35d6fda7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:20:40.232Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"d5af6719-eb7a-4618-8828-fc8a35d6fda7","name":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:20:40.232Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"d5af6719-eb7a-4618-8828-fc8a35d6fda7","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:20:40.232Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"d5af6719-eb7a-4618-8828-fc8a35d6fda7"}
{"level":"info","ts":"2026-11-03T15:20:40.232Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"d5af6719-eb7a-4618-8828-fc8a35d6fda7","RayCluster":"fail-sgjqb","State":""}
{"level":"info","ts":"2026-11-03T15:20:40.232Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"d5af6719-eb7a-4618-8828-fc8a35d6fda7","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:20:40.981Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"c1182856-065c-49bf-92ce-b456a151b6fd","cluster":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:20:43.233Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"74f25512-9157-4f21-944d-360a14e5c9e9","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:20:43.233Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"74f25512-9157-4f21-944d-360a14e5c9e9","name":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:20:43.233Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"74f25512-9157-4f21-944d-360a14e5c9e9","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:20:43.233Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"74f25512-9157-4f21-944d-360a14e5c9e9"}
{"level":"info","ts":"2026-11-03T15:20:43.233Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"74f25512-9157-4f21-944d-360a14e5c9e9","RayCluster":"fail-sgjqb","State":""}
{"level":"info","ts":"2026-11-03T15:20:43.233Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"74f25512-9157-4f21-944d-360a14e5c9e9","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:20:46.234Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"76b0bc22-cae5-4985-8750-dede8f57ccd5","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:20:46.234Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"76b0bc22-cae5-4985-8750-dede8f57ccd5","name":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:20:46.234Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"76b0bc22-cae5-4985-8750-dede8f57ccd5","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:20:46.234Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"76b0bc22-cae5-4985-8750-dede8f57ccd5"}
{"level":"info","ts":"2026-11-03T15:20:46.234Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"76b0bc22-cae5-4985-8750-dede8f57ccd5","RayCluster":"fail-sgjqb","State":""}
{"level":"info","ts":"2026-11-03T15:20:46.234Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"76b0bc22-cae5-4985-8750-dede8f57ccd5","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:20:49.235Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"d2be405b-d47e-41ff-bf33-b65413b94f6d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:20:49.235Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"d2be405b-d47e-41ff-bf33-b65413b94f6d","name":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:20:49.235Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"d2be405b-d47e-41ff-bf33-b65413b94f6d","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:20:49.235Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"d2be405b-d47e-41ff-bf33-b65413b94f6d"}
{"level":"info","ts":"2026-11-03T15:20:49.235Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"d2be405b-d47e-41ff-bf33-b65413b94f6d","RayCluster":"fail-sgjqb","State":""}
{"level":"info","ts":"2026-11-03T15:20:49.235Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"d2be405b-d47e-41ff-bf33-b65413b94f6d","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:20:52.236Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"04c06319-52d2-4674-ba48-bb6a884df607","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:20:52.236Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"04c06319-52d2-4674-ba48-bb6a884df607","name":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:20:52.236Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"04c06319-52d2-4674-ba48-bb6a884df607","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:20:52.236Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"04c06319-52d2-4674-ba48-bb6a884df607"}
{"level":"info","ts":"2026-11-03T15:20:52.236Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"04c06319-52d2-4674-ba48-bb6a884df607","RayCluster":"fail-sgjqb","State":""}
{"level":"info","ts":"2026-11-03T15:20:52.236Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"04c06319-52d2-4674-ba48-bb6a884df607","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:20:55.237Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"cd9a5adb-3559-49f9-880f-7cf6e10a3615","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:20:55.237Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"cd9a5adb-3559-49f9-880f-7cf6e10a3615","name":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:20:55.237Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"cd9a5adb-3559-49f9-880f-7cf6e10a3615","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:20:55.237Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"cd9a5adb-3559-49f9-880f-7cf6e10a3615"}
{"level":"info","ts":"2026-11-03T15:20:55.237Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"cd9a5adb-3559-49f9-880f-7cf6e10a3615","RayCluster":"fail-sgjqb","State":""}
{"level":"info","ts":"2026-11-03T15:20:55.237Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"cd9a5adb-3559-49f9-880f-7cf6e10a3615","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:20:58.238Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"61335d4c-e8b9-4ee4-b0e0-41bcf2f05422","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:20:58.238Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"61335d4c-e8b9-4ee4-b0e0-41bcf2f05422","name":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:20:58.238Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"61335d4c-e8b9-4ee4-b0e0-41bcf2f05422","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:20:58.238Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"61335d4c-e8b9-4ee4-b0e0-41bcf2f05422"}
{"level":"info","ts":"2026-11-03T15:20:58.238Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"61335d4c-e8b9-4ee4-b0e0-41bcf2f05422","RayCluster":"fail-sgjqb","State":""}
{"level":"info","ts":"2026-11-03T15:20:58.238Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"61335d4c-e8b9-4ee4-b0e0-41bcf2f05422","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:21:01.239Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"2d1ce6b5-b08c-4597-b6b4-0034204a615f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:21:01.239Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"2d1ce6b5-b08c-4597-b6b4-0034204a615f","name":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:21:01.239Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"2d1ce6b5-b08c-4597-b6b4-0034204a615f","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:21:01.239Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"2d1ce6b5-b08c-4597-b6b4-0034204a615f"}
{"level":"info","ts":"2026-11-03T15:21:01.239Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"2d1ce6b5-b08c-4597-b6b4-0034204a615f","RayCluster":"fail-sgjqb","State":""}
{"level":"info","ts":"2026-11-03T15:21:01.239Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"2d1ce6b5-b08c-4597-b6b4-0034204a615f","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:21:04.240Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"144ec931-2eeb-4187-b104-3d59b2b4ed84","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:21:04.240Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"144ec931-2eeb-4187-b104-3d59b2b4ed84","name":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:21:04.240Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"144ec931-2eeb-4187-b104-3d59b2b4ed84","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:21:04.240Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"144ec931-2eeb-4187-b104-3d59b2b4ed84"}
{"level":"info","ts":"2026-11-03T15:21:04.240Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"144ec931-2eeb-4187-b104-3d59b2b4ed84","RayCluster":"fail-sgjqb","State":""}
{"level":"info","ts":"2026-11-03T15:21:04.240Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"144ec931-2eeb-4187-b104-3d59b2b4ed84","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:21:07.241Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"1c6e2b3f-73e2-43eb-95eb-aaf3d045256a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:21:07.241Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"1c6e2b3f-73e2-43eb-95eb-aaf3d045256a","name":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:21:07.241Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"1c6e2b3f-73e2-43eb-95eb-aaf3d045256a","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:21:07.241Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"1c6e2b3f-73e2-43eb-95eb-aaf3d045256a"}
{"level":"info","ts":"2026-11-03T15:21:07.241Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"1c6e2b3f-73e2-43eb-95eb-aaf3d045256a","RayCluster":"fail-sgjqb","State":""}
{"level":"info","ts":"2026-11-03T15:21:07.241Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"1c6e2b3f-73e2-43eb-95eb-aaf3d045256a","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:21:10.242Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"77ea26ff-9fd0-4acf-a4f2-79ea16f870b1","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:21:10.242Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"77ea26ff-9fd0-4acf-a4f2-79ea16f870b1","name":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:21:10.242Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"77ea26ff-9fd0-4acf-a4f2-79ea16f870b1","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:21:10.242Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"77ea26ff-9fd0-4acf-a4f2-79ea16f870b1"}
{"level":"info","ts":"2026-11-03T15:21:10.242Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"77ea26ff-9fd0-4acf-a4f2-79ea16f870b1","RayCluster":"fail-sgjqb","State":""}
{"level":"info","ts":"2026-11-03T15:21:10.242Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"77ea26ff-9fd0-4acf-a4f2-79ea16f870b1","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:21:13.243Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"367eb775-ffeb-450f-8440-252edc49ed3e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:21:13.243Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"367eb775-ffeb-450f-8440-252edc49ed3e","name":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:21:13.243Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"367eb775-ffeb-450f-8440-252edc49ed3e","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:21:13.243Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"367eb775-ffeb-450f-8440-252edc49ed3e"}
{"level":"info","ts":"2026-11-03T15:21:13.243Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"367eb775-ffeb-450f-8440-252edc49ed3e","RayCluster":"fail-sgjqb","State":""}
{"level":"info","ts":"2026-11-03T15:21:13.243Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"367eb775-ffeb-450f-8440-252edc49ed3e","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:21:16.244Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"f8c8fce9-ffbc-419a-a186-4faa6662319b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:21:16.244Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"f8c8fce9-ffbc-419a-a186-4faa6662319b","name":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:21:16.244Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"f8c8fce9-ffbc-419a-a186-4faa6662319b","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:21:16.244Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"f8c8fce9-ffbc-419a-a186-4faa6662319b"}
{"level":"info","ts":"2026-11-03T15:21:16.244Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"f8c8fce9-ffbc-419a-a186-4faa6662319b","RayCluster":"fail-sgjqb","State":""}
{"level":"info","ts":"2026-11-03T15:21:16.244Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"f8c8fce9-ffbc-419a-a186-4faa6662319b","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:21:19.245Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"7233eef3-9488-41cb-8bcc-dfbf7c2ed732","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:21:19.245Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"7233eef3-9488-41cb-8bcc-dfbf7c2ed732","name":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:21:19.245Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"7233eef3-9488-41cb-8bcc-dfbf7c2ed732","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:21:19.245Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"7233eef3-9488-41cb-8bcc-dfbf7c2ed732"}
{"level":"info","ts":"2026-11-03T15:21:19.245Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"7233eef3-9488-41cb-8bcc-dfbf7c2ed732","RayCluster":"fail-sgjqb","State":""}
{"level":"info","ts":"2026-11-03T15:21:19.245Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"7233eef3-9488-41cb-8bcc-dfbf7c2ed732","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:21:22.246Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"8e439c89-71ac-4507-8d43-18176bca1928","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:21:22.246Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"8e439c89-71ac-4507-8d43-18176bca1928","name":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:21:22.246Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"8e439c89-71ac-4507-8d43-18176bca1928","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:21:22.246Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"8e439c89-71ac-4507-8d43-18176bca1928"}
{"level":"info","ts":"2026-11-03T15:21:22.246Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"8e439c89-71ac-4507-8d43-18176bca1928","RayCluster":"fail-sgjqb","State":""}
{"level":"info","ts":"2026-11-03T15:21:22.246Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"8e439c89-71ac-4507-8d43-18176bca1928","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:21:25.246Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"353ba6d8-b584-42d6-9573-262db016bc42","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:21:25.246Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"353ba6d8-b584-42d6-9573-262db016bc42","name":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:21:25.246Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"353ba6d8-b584-42d6-9573-262db016bc42","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:21:25.247Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"353ba6d8-b584-42d6-9573-262db016bc42"}
{"level":"info","ts":"2026-11-03T15:21:25.247Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"353ba6d8-b584-42d6-9573-262db016bc42","RayCluster":"fail-sgjqb","State":""}
{"level":"info","ts":"2026-11-03T15:21:25.247Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"353ba6d8-b584-42d6-9573-262db016bc42","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:21:28.248Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"6dd75777-e6e1-40ca-9b6f-b38ab59ce486","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:21:28.248Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"6dd75777-e6e1-40ca-9b6f-b38ab59ce486","name":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:21:28.248Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"6dd75777-e6e1-40ca-9b6f-b38ab59ce486","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:21:28.248Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"6dd75777-e6e1-40ca-9b6f-b38ab59ce486"}
{"level":"info","ts":"2026-11-03T15:21:28.248Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"6dd75777-e6e1-40ca-9b6f-b38ab59ce486","RayCluster":"fail-sgjqb","State":""}
{"level":"info","ts":"2026-11-03T15:21:28.248Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"6dd75777-e6e1-40ca-9b6f-b38ab59ce486","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:21:30.453Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"17ddb512-f969-4cc8-ad39-1edc5891e4cb"}
{"level":"info","ts":"2026-11-03T15:21:30.454Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"17ddb512-f969-4cc8-ad39-1edc5891e4cb"}
{"level":"info","ts":"2026-11-03T15:21:30.454Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"17ddb512-f969-4cc8-ad39-1edc5891e4cb","1 head service found":"fail-sgjqb-head-svc"}
{"level":"info","ts":"2026-11-03T15:21:30.454Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"17ddb512-f969-4cc8-ad39-1edc5891e4cb","Found 1 head Pod":"fail-sgjqb-head-8pdmr","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:21:30.454Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"17ddb512-f969-4cc8-ad39-1edc5891e4cb","head Pod":"fail-sgjqb-head-8pdmr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-sgjqb-head-8pdmr. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:21:30.454Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"17ddb512-f969-4cc8-ad39-1edc5891e4cb","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:21:30.454Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"17ddb512-f969-4cc8-ad39-1edc5891e4cb","worker Pod":"fail-sgjqb-small-group-worker-k4f2l","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-sgjqb-small-group-worker-k4f2l. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:21:30.454Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"17ddb512-f969-4cc8-ad39-1edc5891e4cb","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:21:30.454Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"17ddb512-f969-4cc8-ad39-1edc5891e4cb","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:21:30.454Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"17ddb512-f969-4cc8-ad39-1edc5891e4cb","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:21:30.454Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"17ddb512-f969-4cc8-ad39-1edc5891e4cb","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:21:30.454Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"17ddb512-f969-4cc8-ad39-1edc5891e4cb","podName":"fail-sgjqb-head-8pdmr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:21:30.454Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"17ddb512-f969-4cc8-ad39-1edc5891e4cb","podName":"fail-sgjqb-head-8pdmr","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:21:30.454Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"17ddb512-f969-4cc8-ad39-1edc5891e4cb","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:21:30.454Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"17ddb512-f969-4cc8-ad39-1edc5891e4cb","seconds":300}
{"level":"info","ts":"2026-11-03T15:21:31.249Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"e5d056b4-75ec-443f-9905-6962762acf02","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:21:31.249Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"e5d056b4-75ec-443f-9905-6962762acf02","name":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:21:31.249Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"e5d056b4-75ec-443f-9905-6962762acf02","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:21:31.249Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"e5d056b4-75ec-443f-9905-6962762acf02"}
{"level":"info","ts":"2026-11-03T15:21:31.249Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"e5d056b4-75ec-443f-9905-6962762acf02","RayCluster":"fail-sgjqb","State":""}
{"level":"info","ts":"2026-11-03T15:21:31.249Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"e5d056b4-75ec-443f-9905-6962762acf02","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:21:31.534Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"d1387efe-623f-482c-a961-2797a91b2a4b"}
{"level":"info","ts":"2026-11-03T15:21:31.534Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"d1387efe-623f-482c-a961-2797a91b2a4b"}
{"level":"info","ts":"2026-11-03T15:21:31.534Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"d1387efe-623f-482c-a961-2797a91b2a4b","1 head service found":"fail-sgjqb-head-svc"}
{"level":"info","ts":"2026-11-03T15:21:31.534Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"d1387efe-623f-482c-a961-2797a91b2a4b","Found 1 head Pod":"fail-sgjqb-head-8pdmr","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:21:31.534Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"d1387efe-623f-482c-a961-2797a91b2a4b","head Pod":"fail-sgjqb-head-8pdmr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-sgjqb-head-8pdmr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:21:31.534Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"d1387efe-623f-482c-a961-2797a91b2a4b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:21:31.534Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"d1387efe-623f-482c-a961-2797a91b2a4b","worker Pod":"fail-sgjqb-small-group-worker-k4f2l","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-sgjqb-small-group-worker-k4f2l. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:21:31.534Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"d1387efe-623f-482c-a961-2797a91b2a4b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:21:31.534Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"d1387efe-623f-482c-a961-2797a91b2a4b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:21:31.534Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"d1387efe-623f-482c-a961-2797a91b2a4b","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:21:31.534Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"d1387efe-623f-482c-a961-2797a91b2a4b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:21:31.534Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"d1387efe-623f-482c-a961-2797a91b2a4b","podName":"fail-sgjqb-small-group-worker-k4f2l","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:21:31.534Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"d1387efe-623f-482c-a961-2797a91b2a4b","podName":"fail-sgjqb-small-group-worker-k4f2l","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:21:31.534Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"d1387efe-623f-482c-a961-2797a91b2a4b","name":"fail-sgjqb","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:21:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:21:31.597Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"180b3b2f-aa0c-4ce6-9f23-8938aef5fd98","cluster":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:21:31.597Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"449868c9-5846-493c-8a36-8a09463b0072","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:21:31.597Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"449868c9-5846-493c-8a36-8a09463b0072","name":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:21:31.597Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"449868c9-5846-493c-8a36-8a09463b0072","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:21:31.597Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"449868c9-5846-493c-8a36-8a09463b0072"}
{"level":"info","ts":"2026-11-03T15:21:31.597Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"449868c9-5846-493c-8a36-8a09463b0072","RayCluster":"fail-sgjqb","State":""}
{"level":"info","ts":"2026-11-03T15:21:31.597Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"449868c9-5846-493c-8a36-8a09463b0072","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:20:10Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:21:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:21:31.597Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"449868c9-5846-493c-8a36-8a09463b0072","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:21:31.659Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"89f60b33-da70-4a13-945a-7dbefcac6364","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:21:31.659Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"89f60b33-da70-4a13-945a-7dbefcac6364","name":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:21:31.659Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"89f60b33-da70-4a13-945a-7dbefcac6364","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:21:31.659Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"89f60b33-da70-4a13-945a-7dbefcac6364"}
{"level":"info","ts":"2026-11-03T15:21:31.659Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"89f60b33-da70-4a13-945a-7dbefcac6364","RayCluster":"fail-sgjqb","State":""}
{"level":"info","ts":"2026-11-03T15:21:31.659Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"89f60b33-da70-4a13-945a-7dbefcac6364","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:21:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:21:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:21:33.597Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"d93cc34f-54ef-4674-9064-35a18d2cf502"}
{"level":"info","ts":"2026-11-03T15:21:33.597Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"d93cc34f-54ef-4674-9064-35a18d2cf502"}
{"level":"info","ts":"2026-11-03T15:21:33.597Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"d93cc34f-54ef-4674-9064-35a18d2cf502","1 head service found":"fail-sgjqb-head-svc"}
{"level":"info","ts":"2026-11-03T15:21:33.597Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"d93cc34f-54ef-4674-9064-35a18d2cf502","Found 1 head Pod":"fail-sgjqb-head-8pdmr","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:21:33.597Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"d93cc34f-54ef-4674-9064-35a18d2cf502","head Pod":"fail-sgjqb-head-8pdmr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-sgjqb-head-8pdmr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:21:33.597Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"d93cc34f-54ef-4674-9064-35a18d2cf502","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:21:33.597Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"d93cc34f-54ef-4674-9064-35a18d2cf502","worker Pod":"fail-sgjqb-small-group-worker-k4f2l","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-sgjqb-small-group-worker-k4f2l. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:21:33.597Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"d93cc34f-54ef-4674-9064-35a18d2cf502","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:21:33.597Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"d93cc34f-54ef-4674-9064-35a18d2cf502","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:21:33.597Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"d93cc34f-54ef-4674-9064-35a18d2cf502","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:21:33.597Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"d93cc34f-54ef-4674-9064-35a18d2cf502","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:21:33.598Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"d93cc34f-54ef-4674-9064-35a18d2cf502","podName":"fail-sgjqb-head-8pdmr","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:20:08Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:21:33.598Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"d93cc34f-54ef-4674-9064-35a18d2cf502","podName":"fail-sgjqb-head-8pdmr","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:20:08Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:21:33.598Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"d93cc34f-54ef-4674-9064-35a18d2cf502","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:21:33.598Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"d93cc34f-54ef-4674-9064-35a18d2cf502","seconds":300}
{"level":"info","ts":"2026-11-03T15:21:34.660Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"8ed80383-d80c-439a-b2e6-8e72fc8c7193","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:21:34.660Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"8ed80383-d80c-439a-b2e6-8e72fc8c7193","name":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:21:34.660Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"8ed80383-d80c-439a-b2e6-8e72fc8c7193","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:21:34.660Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"8ed80383-d80c-439a-b2e6-8e72fc8c7193"}
{"level":"info","ts":"2026-11-03T15:21:34.660Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"8ed80383-d80c-439a-b2e6-8e72fc8c7193","RayCluster":"fail-sgjqb","State":""}
{"level":"info","ts":"2026-11-03T15:21:34.660Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"8ed80383-d80c-439a-b2e6-8e72fc8c7193","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:21:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:21:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:21:35.528Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"49bec41a-0e57-414e-b765-486521042731"}
{"level":"info","ts":"2026-11-03T15:21:35.528Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"49bec41a-0e57-414e-b765-486521042731"}
{"level":"info","ts":"2026-11-03T15:21:35.528Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"49bec41a-0e57-414e-b765-486521042731","1 head service found":"fail-sgjqb-head-svc"}
{"level":"info","ts":"2026-11-03T15:21:35.528Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"49bec41a-0e57-414e-b765-486521042731","Found 1 head Pod":"fail-sgjqb-head-8pdmr","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:21:35.528Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"49bec41a-0e57-414e-b765-486521042731","head Pod":"fail-sgjqb-head-8pdmr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-sgjqb-head-8pdmr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:21:35.528Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"49bec41a-0e57-414e-b765-486521042731","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:21:35.528Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"49bec41a-0e57-414e-b765-486521042731","worker Pod":"fail-sgjqb-small-group-worker-k4f2l","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-sgjqb-small-group-worker-k4f2l. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:21:35.528Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"49bec41a-0e57-414e-b765-486521042731","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:21:35.528Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"49bec41a-0e57-414e-b765-486521042731","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:21:35.528Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"49bec41a-0e57-414e-b765-486521042731","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:21:35.528Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"49bec41a-0e57-414e-b765-486521042731","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:21:35.528Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"49bec41a-0e57-414e-b765-486521042731","podName":"fail-sgjqb-head-8pdmr","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:20:08Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:21:35.528Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"49bec41a-0e57-414e-b765-486521042731","podName":"fail-sgjqb-head-8pdmr","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:20:08Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:21:35.529Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"49bec41a-0e57-414e-b765-486521042731","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:21:35.529Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"49bec41a-0e57-414e-b765-486521042731","seconds":300}
{"level":"info","ts":"2026-11-03T15:21:36.520Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"7ba9044d-2451-45ae-86e8-4c1e3cc71560"}
{"level":"info","ts":"2026-11-03T15:21:36.520Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"7ba9044d-2451-45ae-86e8-4c1e3cc71560"}
{"level":"info","ts":"2026-11-03T15:21:36.521Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"7ba9044d-2451-45ae-86e8-4c1e3cc71560","1 head service found":"fail-sgjqb-head-svc"}
{"level":"info","ts":"2026-11-03T15:21:36.521Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"7ba9044d-2451-45ae-86e8-4c1e3cc71560","Found 1 head Pod":"fail-sgjqb-head-8pdmr","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:21:36.521Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"7ba9044d-2451-45ae-86e8-4c1e3cc71560","head Pod":"fail-sgjqb-head-8pdmr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-sgjqb-head-8pdmr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:21:36.521Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"7ba9044d-2451-45ae-86e8-4c1e3cc71560","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:21:36.521Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"7ba9044d-2451-45ae-86e8-4c1e3cc71560","worker Pod":"fail-sgjqb-small-group-worker-k4f2l","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-sgjqb-small-group-worker-k4f2l. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:21:36.521Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"7ba9044d-2451-45ae-86e8-4c1e3cc71560","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:21:36.521Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"7ba9044d-2451-45ae-86e8-4c1e3cc71560","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:21:36.521Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"7ba9044d-2451-45ae-86e8-4c1e3cc71560","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:21:36.521Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"7ba9044d-2451-45ae-86e8-4c1e3cc71560","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:21:36.522Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"7ba9044d-2451-45ae-86e8-4c1e3cc71560","podName":"fail-sgjqb-small-group-worker-k4f2l","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:21:36.522Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"7ba9044d-2451-45ae-86e8-4c1e3cc71560","podName":"fail-sgjqb-small-group-worker-k4f2l","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:21:36.522Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"7ba9044d-2451-45ae-86e8-4c1e3cc71560","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:21:36.522Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"7ba9044d-2451-45ae-86e8-4c1e3cc71560","seconds":300}
{"level":"info","ts":"2026-11-03T15:21:37.661Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"eee6d379-203c-4a22-ba6d-fbb44d000aeb","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:21:37.661Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"eee6d379-203c-4a22-ba6d-fbb44d000aeb","name":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:21:37.661Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"eee6d379-203c-4a22-ba6d-fbb44d000aeb","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:21:37.662Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"eee6d379-203c-4a22-ba6d-fbb44d000aeb"}
{"level":"info","ts":"2026-11-03T15:21:37.662Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"eee6d379-203c-4a22-ba6d-fbb44d000aeb","RayCluster":"fail-sgjqb","State":""}
{"level":"info","ts":"2026-11-03T15:21:37.662Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"eee6d379-203c-4a22-ba6d-fbb44d000aeb","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:21:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:21:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:21:40.662Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"fc4fbc06-5f94-4cd7-8746-6d814d439c03","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:21:40.662Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"fc4fbc06-5f94-4cd7-8746-6d814d439c03","name":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:21:40.662Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"fc4fbc06-5f94-4cd7-8746-6d814d439c03","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:21:40.662Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"fc4fbc06-5f94-4cd7-8746-6d814d439c03"}
{"level":"info","ts":"2026-11-03T15:21:40.662Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"fc4fbc06-5f94-4cd7-8746-6d814d439c03","RayCluster":"fail-sgjqb","State":""}
{"level":"info","ts":"2026-11-03T15:21:40.662Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"fc4fbc06-5f94-4cd7-8746-6d814d439c03","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:21:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:21:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:21:42.554Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"ce99642b-c756-45b3-a8ba-8b031f73ddd4"}
{"level":"info","ts":"2026-11-03T15:21:42.554Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"ce99642b-c756-45b3-a8ba-8b031f73ddd4"}
{"level":"info","ts":"2026-11-03T15:21:42.554Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"ce99642b-c756-45b3-a8ba-8b031f73ddd4","1 head service found":"fail-sgjqb-head-svc"}
{"level":"info","ts":"2026-11-03T15:21:42.554Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"ce99642b-c756-45b3-a8ba-8b031f73ddd4","Found 1 head Pod":"fail-sgjqb-head-8pdmr","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:21:42.554Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"ce99642b-c756-45b3-a8ba-8b031f73ddd4","head Pod":"fail-sgjqb-head-8pdmr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-sgjqb-head-8pdmr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:21:42.554Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"ce99642b-c756-45b3-a8ba-8b031f73ddd4","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:21:42.554Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"ce99642b-c756-45b3-a8ba-8b031f73ddd4","worker Pod":"fail-sgjqb-small-group-worker-k4f2l","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-sgjqb-small-group-worker-k4f2l. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:21:42.554Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"ce99642b-c756-45b3-a8ba-8b031f73ddd4","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:21:42.554Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"ce99642b-c756-45b3-a8ba-8b031f73ddd4","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:21:42.554Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"ce99642b-c756-45b3-a8ba-8b031f73ddd4","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:21:42.554Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"ce99642b-c756-45b3-a8ba-8b031f73ddd4","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:21:42.554Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"ce99642b-c756-45b3-a8ba-8b031f73ddd4","podName":"fail-sgjqb-small-group-worker-k4f2l","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:21:42.554Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"ce99642b-c756-45b3-a8ba-8b031f73ddd4","podName":"fail-sgjqb-small-group-worker-k4f2l","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:21:42.554Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"ce99642b-c756-45b3-a8ba-8b031f73ddd4","name":"fail-sgjqb","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:21:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:21:42Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:21:42Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:21:42.615Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"a9212989-fe34-44a5-950b-41cc51bc98ae","cluster":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:21:42.615Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"d642d836-31b5-4b8a-ad6b-557417479bbf","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:21:42.615Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"d642d836-31b5-4b8a-ad6b-557417479bbf","name":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:21:42.615Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"d642d836-31b5-4b8a-ad6b-557417479bbf","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:21:42.616Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"d642d836-31b5-4b8a-ad6b-557417479bbf"}
{"level":"info","ts":"2026-11-03T15:21:42.616Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"d642d836-31b5-4b8a-ad6b-557417479bbf","RayCluster":"fail-sgjqb","State":""}
{"level":"info","ts":"2026-11-03T15:21:42.616Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"d642d836-31b5-4b8a-ad6b-557417479bbf","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:21:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:21:42Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:21:42Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:21:42.616Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"d642d836-31b5-4b8a-ad6b-557417479bbf","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:21:42.677Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"68ebedea-18f0-426e-9758-74d76067c6d8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:21:42.677Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"68ebedea-18f0-426e-9758-74d76067c6d8","name":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:21:42.677Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"68ebedea-18f0-426e-9758-74d76067c6d8","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:21:42.677Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"68ebedea-18f0-426e-9758-74d76067c6d8"}
{"level":"info","ts":"2026-11-03T15:21:42.677Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"68ebedea-18f0-426e-9758-74d76067c6d8","RayCluster":"fail-sgjqb","State":""}
{"level":"info","ts":"2026-11-03T15:21:42.677Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"68ebedea-18f0-426e-9758-74d76067c6d8","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:21:42Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:21:42Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:21:42Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:21:42Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:21:44.616Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"0369618f-9a1b-40df-bc40-732d7d0ea115"}
{"level":"info","ts":"2026-11-03T15:21:44.616Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"0369618f-9a1b-40df-bc40-732d7d0ea115"}
{"level":"info","ts":"2026-11-03T15:21:44.616Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"0369618f-9a1b-40df-bc40-732d7d0ea115","1 head service found":"fail-sgjqb-head-svc"}
{"level":"info","ts":"2026-11-03T15:21:44.616Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"0369618f-9a1b-40df-bc40-732d7d0ea115","Found 1 head Pod":"fail-sgjqb-head-8pdmr","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:21:44.616Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"0369618f-9a1b-40df-bc40-732d7d0ea115","head Pod":"fail-sgjqb-head-8pdmr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-sgjqb-head-8pdmr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:21:44.616Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"0369618f-9a1b-40df-bc40-732d7d0ea115","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:21:44.616Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"0369618f-9a1b-40df-bc40-732d7d0ea115","worker Pod":"fail-sgjqb-small-group-worker-k4f2l","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-sgjqb-small-group-worker-k4f2l. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:21:44.616Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"0369618f-9a1b-40df-bc40-732d7d0ea115","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:21:44.616Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"0369618f-9a1b-40df-bc40-732d7d0ea115","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:21:44.616Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"0369618f-9a1b-40df-bc40-732d7d0ea115","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:21:44.616Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"0369618f-9a1b-40df-bc40-732d7d0ea115","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:21:44.616Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"0369618f-9a1b-40df-bc40-732d7d0ea115","podName":"fail-sgjqb-small-group-worker-k4f2l","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:21:44.617Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"0369618f-9a1b-40df-bc40-732d7d0ea115","podName":"fail-sgjqb-small-group-worker-k4f2l","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:21:44.617Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"0369618f-9a1b-40df-bc40-732d7d0ea115","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:21:44.617Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"0369618f-9a1b-40df-bc40-732d7d0ea115","seconds":300}
{"level":"info","ts":"2026-11-03T15:21:45.554Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"429acd11-57d3-433d-b25c-3dc262f7b1a8"}
{"level":"info","ts":"2026-11-03T15:21:45.554Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"429acd11-57d3-433d-b25c-3dc262f7b1a8"}
{"level":"info","ts":"2026-11-03T15:21:45.554Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"429acd11-57d3-433d-b25c-3dc262f7b1a8","1 head service found":"fail-sgjqb-head-svc"}
{"level":"info","ts":"2026-11-03T15:21:45.554Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"429acd11-57d3-433d-b25c-3dc262f7b1a8","Found 1 head Pod":"fail-sgjqb-head-8pdmr","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:21:45.554Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"429acd11-57d3-433d-b25c-3dc262f7b1a8","head Pod":"fail-sgjqb-head-8pdmr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-sgjqb-head-8pdmr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:21:45.554Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"429acd11-57d3-433d-b25c-3dc262f7b1a8","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:21:45.554Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"429acd11-57d3-433d-b25c-3dc262f7b1a8","worker Pod":"fail-sgjqb-small-group-worker-k4f2l","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-sgjqb-small-group-worker-k4f2l. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:21:45.554Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"429acd11-57d3-433d-b25c-3dc262f7b1a8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:21:45.554Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"429acd11-57d3-433d-b25c-3dc262f7b1a8","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:21:45.554Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"429acd11-57d3-433d-b25c-3dc262f7b1a8","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:21:45.554Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"429acd11-57d3-433d-b25c-3dc262f7b1a8","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:21:45.555Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"429acd11-57d3-433d-b25c-3dc262f7b1a8","podName":"fail-sgjqb-small-group-worker-k4f2l","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:21:45.555Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"429acd11-57d3-433d-b25c-3dc262f7b1a8","podName":"fail-sgjqb-small-group-worker-k4f2l","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:21:45.555Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"429acd11-57d3-433d-b25c-3dc262f7b1a8","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:21:45.555Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"429acd11-57d3-433d-b25c-3dc262f7b1a8","seconds":300}
{"level":"info","ts":"2026-11-03T15:21:45.678Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"aacc1430-ab88-4984-811c-6a543bbbf4f9","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:21:45.678Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"aacc1430-ab88-4984-811c-6a543bbbf4f9","name":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:21:45.678Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"aacc1430-ab88-4984-811c-6a543bbbf4f9","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:21:45.678Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"aacc1430-ab88-4984-811c-6a543bbbf4f9"}
{"level":"info","ts":"2026-11-03T15:21:45.678Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"aacc1430-ab88-4984-811c-6a543bbbf4f9","RayCluster":"fail-sgjqb","State":""}
{"level":"info","ts":"2026-11-03T15:21:45.678Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"aacc1430-ab88-4984-811c-6a543bbbf4f9","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:21:42Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:21:42Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:21:42Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:21:42Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:21:46.508Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"a358419a-260e-401a-9df3-5502e0d54e27"}
{"level":"info","ts":"2026-11-03T15:21:46.508Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"a358419a-260e-401a-9df3-5502e0d54e27"}
{"level":"info","ts":"2026-11-03T15:21:46.508Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"a358419a-260e-401a-9df3-5502e0d54e27","1 head service found":"fail-sgjqb-head-svc"}
{"level":"info","ts":"2026-11-03T15:21:46.508Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"a358419a-260e-401a-9df3-5502e0d54e27","Found 1 head Pod":"fail-sgjqb-head-8pdmr","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:21:46.508Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"a358419a-260e-401a-9df3-5502e0d54e27","head Pod":"fail-sgjqb-head-8pdmr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-sgjqb-head-8pdmr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:21:46.508Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"a358419a-260e-401a-9df3-5502e0d54e27","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:21:46.508Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"a358419a-260e-401a-9df3-5502e0d54e27","worker Pod":"fail-sgjqb-small-group-worker-k4f2l","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-sgjqb-small-group-worker-k4f2l. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:21:46.508Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"a358419a-260e-401a-9df3-5502e0d54e27","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:21:46.508Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"a358419a-260e-401a-9df3-5502e0d54e27","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:21:46.508Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"a358419a-260e-401a-9df3-5502e0d54e27","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:21:46.508Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"a358419a-260e-401a-9df3-5502e0d54e27","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:21:46.508Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"a358419a-260e-401a-9df3-5502e0d54e27","podName":"fail-sgjqb-small-group-worker-k4f2l","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:20:08Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:21:46.508Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"a358419a-260e-401a-9df3-5502e0d54e27","podName":"fail-sgjqb-small-group-worker-k4f2l","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:20:08Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:21:46.508Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"a358419a-260e-401a-9df3-5502e0d54e27","name":"fail-sgjqb","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:21:42Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:21:42Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:21:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:21:42Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:21:46.570Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"590d46f1-fd20-462f-9f4a-4c2c481f81cf","cluster":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:21:46.570Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"205c03b5-7e20-49c1-8bdd-1ae0d8fd1c4f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:21:46.570Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"205c03b5-7e20-49c1-8bdd-1ae0d8fd1c4f","name":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:21:46.570Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"205c03b5-7e20-49c1-8bdd-1ae0d8fd1c4f","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:21:46.570Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"205c03b5-7e20-49c1-8bdd-1ae0d8fd1c4f"}
{"level":"info","ts":"2026-11-03T15:21:46.570Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"205c03b5-7e20-49c1-8bdd-1ae0d8fd1c4f","RayCluster":"fail-sgjqb","State":""}
{"level":"info","ts":"2026-11-03T15:21:46.570Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"205c03b5-7e20-49c1-8bdd-1ae0d8fd1c4f","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:21:42Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:21:42Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:21:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:21:42Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:21:46.570Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"205c03b5-7e20-49c1-8bdd-1ae0d8fd1c4f","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:21:46.632Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"02585a99-4e38-4491-accc-be99b7e3c255","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:21:46.632Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"02585a99-4e38-4491-accc-be99b7e3c255","name":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:21:46.632Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"02585a99-4e38-4491-accc-be99b7e3c255","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:21:46.632Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"02585a99-4e38-4491-accc-be99b7e3c255"}
{"level":"info","ts":"2026-11-03T15:21:46.632Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"02585a99-4e38-4491-accc-be99b7e3c255","RayCluster":"fail-sgjqb","State":""}
{"level":"info","ts":"2026-11-03T15:21:46.632Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"02585a99-4e38-4491-accc-be99b7e3c255","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:21:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:21:42Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:21:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:21:42Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:21:48.570Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"33094ed9-2aeb-4e40-b67c-f144ce1e80b9"}
{"level":"info","ts":"2026-11-03T15:21:48.570Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"33094ed9-2aeb-4e40-b67c-f144ce1e80b9"}
{"level":"info","ts":"2026-11-03T15:21:48.570Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"33094ed9-2aeb-4e40-b67c-f144ce1e80b9","1 head service found":"fail-sgjqb-head-svc"}
{"level":"info","ts":"2026-11-03T15:21:48.570Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"33094ed9-2aeb-4e40-b67c-f144ce1e80b9","Found 1 head Pod":"fail-sgjqb-head-8pdmr","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:21:48.570Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"33094ed9-2aeb-4e40-b67c-f144ce1e80b9","head Pod":"fail-sgjqb-head-8pdmr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-sgjqb-head-8pdmr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:21:48.570Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"33094ed9-2aeb-4e40-b67c-f144ce1e80b9","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:21:48.570Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"33094ed9-2aeb-4e40-b67c-f144ce1e80b9","worker Pod":"fail-sgjqb-small-group-worker-k4f2l","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-sgjqb-small-group-worker-k4f2l. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:21:48.570Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"33094ed9-2aeb-4e40-b67c-f144ce1e80b9","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:21:48.570Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"33094ed9-2aeb-4e40-b67c-f144ce1e80b9","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:21:48.570Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"33094ed9-2aeb-4e40-b67c-f144ce1e80b9","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:21:48.570Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"33094ed9-2aeb-4e40-b67c-f144ce1e80b9","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:21:48.570Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"33094ed9-2aeb-4e40-b67c-f144ce1e80b9","podName":"fail-sgjqb-small-group-worker-k4f2l","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:20:08Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:21:48.570Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"33094ed9-2aeb-4e40-b67c-f144ce1e80b9","podName":"fail-sgjqb-small-group-worker-k4f2l","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:20:08Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:21:48.571Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"33094ed9-2aeb-4e40-b67c-f144ce1e80b9","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:21:48.571Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"33094ed9-2aeb-4e40-b67c-f144ce1e80b9","seconds":300}
{"level":"info","ts":"2026-11-03T15:21:49.633Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"d798210b-8338-4497-ad81-f3f358108efe","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:21:49.633Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"d798210b-8338-4497-ad81-f3f358108efe","name":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:21:49.633Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"d798210b-8338-4497-ad81-f3f358108efe","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:21:49.633Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"d798210b-8338-4497-ad81-f3f358108efe"}
{"level":"info","ts":"2026-11-03T15:21:49.633Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"d798210b-8338-4497-ad81-f3f358108efe","RayCluster":"fail-sgjqb","State":""}
{"level":"info","ts":"2026-11-03T15:21:49.633Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"d798210b-8338-4497-ad81-f3f358108efe","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:21:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:21:42Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:21:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:21:42Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:21:52.633Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"b77ac67d-a63e-4b0b-b625-dc82fa49f9e1","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:21:52.633Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"b77ac67d-a63e-4b0b-b625-dc82fa49f9e1","name":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:21:52.633Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"b77ac67d-a63e-4b0b-b625-dc82fa49f9e1","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:21:52.634Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"b77ac67d-a63e-4b0b-b625-dc82fa49f9e1"}
{"level":"info","ts":"2026-11-03T15:21:52.634Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"b77ac67d-a63e-4b0b-b625-dc82fa49f9e1","RayCluster":"fail-sgjqb","State":""}
{"level":"info","ts":"2026-11-03T15:21:52.634Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"b77ac67d-a63e-4b0b-b625-dc82fa49f9e1","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:21:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:21:42Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:21:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:21:42Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:21:55.634Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"74bb0bbc-d7a4-4112-a7e6-0130d0e84580","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:21:55.634Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"74bb0bbc-d7a4-4112-a7e6-0130d0e84580","name":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:21:55.634Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"74bb0bbc-d7a4-4112-a7e6-0130d0e84580","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:21:55.634Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"74bb0bbc-d7a4-4112-a7e6-0130d0e84580"}
{"level":"info","ts":"2026-11-03T15:21:55.634Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"74bb0bbc-d7a4-4112-a7e6-0130d0e84580","RayCluster":"fail-sgjqb","State":""}
{"level":"info","ts":"2026-11-03T15:21:55.634Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"74bb0bbc-d7a4-4112-a7e6-0130d0e84580","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:21:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:21:42Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:21:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:21:42Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:21:57.563Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"5319f196-5ea3-4509-a9c6-e5fd833a6fa5"}
{"level":"info","ts":"2026-11-03T15:21:57.563Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"5319f196-5ea3-4509-a9c6-e5fd833a6fa5"}
{"level":"info","ts":"2026-11-03T15:21:57.563Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"5319f196-5ea3-4509-a9c6-e5fd833a6fa5","1 head service found":"fail-sgjqb-head-svc"}
{"level":"info","ts":"2026-11-03T15:21:57.564Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"5319f196-5ea3-4509-a9c6-e5fd833a6fa5","Found 1 head Pod":"fail-sgjqb-head-8pdmr","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:21:57.564Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"5319f196-5ea3-4509-a9c6-e5fd833a6fa5","head Pod":"fail-sgjqb-head-8pdmr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-sgjqb-head-8pdmr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:21:57.564Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"5319f196-5ea3-4509-a9c6-e5fd833a6fa5","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:21:57.564Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"5319f196-5ea3-4509-a9c6-e5fd833a6fa5","worker Pod":"fail-sgjqb-small-group-worker-k4f2l","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-sgjqb-small-group-worker-k4f2l. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:21:57.564Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"5319f196-5ea3-4509-a9c6-e5fd833a6fa5","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:21:57.564Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"5319f196-5ea3-4509-a9c6-e5fd833a6fa5","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:21:57.564Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"5319f196-5ea3-4509-a9c6-e5fd833a6fa5","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:21:57.564Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"5319f196-5ea3-4509-a9c6-e5fd833a6fa5","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:21:57.564Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"5319f196-5ea3-4509-a9c6-e5fd833a6fa5","name":"fail-sgjqb","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:21:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:21:42Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:21:57Z","stateTransitionTimes":{"ready":"2026-11-03T15:21:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:21:42Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:21:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:21:57.625Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"e4a1d2af-a9a8-443b-82a8-21cfa13ff293","cluster":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:21:57.625Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"120e40f6-0da5-42fd-95b4-74b4c5e1b042","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:21:57.625Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"120e40f6-0da5-42fd-95b4-74b4c5e1b042","name":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:21:57.625Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"120e40f6-0da5-42fd-95b4-74b4c5e1b042","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:21:57.626Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"120e40f6-0da5-42fd-95b4-74b4c5e1b042"}
{"level":"info","ts":"2026-11-03T15:21:57.689Z","logger":"controllers.RayJob","msg":"Created submitter Kubernetes Job for RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"120e40f6-0da5-42fd-95b4-74b4c5e1b042","Kubernetes Job":"fail"}
{"level":"info","ts":"2026-11-03T15:21:57.689Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"120e40f6-0da5-42fd-95b4-74b4c5e1b042","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:21:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:21:42Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","dashboardURL":"fail-sgjqb-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:21:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:21:42Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:21:57.689Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"120e40f6-0da5-42fd-95b4-74b4c5e1b042","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-11-03T15:21:57.768Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"f883f12d-596a-44f9-ad39-b7685139e3c6","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:21:57.768Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"f883f12d-596a-44f9-ad39-b7685139e3c6","name":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:21:57.768Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"f883f12d-596a-44f9-ad39-b7685139e3c6","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:21:57.768Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"f883f12d-596a-44f9-ad39-b7685139e3c6"}
{"level":"info","ts":"2026-11-03T15:21:57.768Z","logger":"controllers.RayJob","msg":"Create a Kubernetes Service","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"f883f12d-596a-44f9-ad39-b7685139e3c6","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:21:57.832Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"68d8f567-b913-47ef-971d-5664493361ca","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:21:57.832Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"68d8f567-b913-47ef-971d-5664493361ca","name":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:21:57.832Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"68d8f567-b913-47ef-971d-5664493361ca","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:21:57.832Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"68d8f567-b913-47ef-971d-5664493361ca"}
{"level":"info","ts":"2026-11-03T15:21:57.832Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"68d8f567-b913-47ef-971d-5664493361ca","rayCluster":"fail-sgjqb","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:21:59.627Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"975fd7b7-2c89-4df3-aafd-2e4bfe10aed8"}
{"level":"info","ts":"2026-11-03T15:21:59.627Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"975fd7b7-2c89-4df3-aafd-2e4bfe10aed8"}
{"level":"info","ts":"2026-11-03T15:21:59.627Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"975fd7b7-2c89-4df3-aafd-2e4bfe10aed8","1 head service found":"fail-sgjqb-head-svc"}
{"level":"info","ts":"2026-11-03T15:21:59.627Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"975fd7b7-2c89-4df3-aafd-2e4bfe10aed8","Found 1 head Pod":"fail-sgjqb-head-8pdmr","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:21:59.627Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"975fd7b7-2c89-4df3-aafd-2e4bfe10aed8","head Pod":"fail-sgjqb-head-8pdmr","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-sgjqb-head-8pdmr. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:21:59.627Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"975fd7b7-2c89-4df3-aafd-2e4bfe10aed8","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:21:59.627Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"975fd7b7-2c89-4df3-aafd-2e4bfe10aed8","worker Pod":"fail-sgjqb-small-group-worker-k4f2l","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-sgjqb-small-group-worker-k4f2l. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:21:59.627Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"975fd7b7-2c89-4df3-aafd-2e4bfe10aed8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:21:59.627Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"975fd7b7-2c89-4df3-aafd-2e4bfe10aed8","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:21:59.627Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"975fd7b7-2c89-4df3-aafd-2e4bfe10aed8","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:21:59.627Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"975fd7b7-2c89-4df3-aafd-2e4bfe10aed8","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:21:59.627Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"975fd7b7-2c89-4df3-aafd-2e4bfe10aed8","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:21:59.627Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"975fd7b7-2c89-4df3-aafd-2e4bfe10aed8","seconds":300}
{"level":"info","ts":"2026-11-03T15:22:00.568Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"a1ebec4b-664c-4ad8-bd83-a6ed45a1c41d","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:22:00.568Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"a1ebec4b-664c-4ad8-bd83-a6ed45a1c41d","name":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:22:00.568Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"a1ebec4b-664c-4ad8-bd83-a6ed45a1c41d","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:22:00.568Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"a1ebec4b-664c-4ad8-bd83-a6ed45a1c41d"}
{"level":"info","ts":"2026-11-03T15:22:00.568Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"a1ebec4b-664c-4ad8-bd83-a6ed45a1c41d","rayCluster":"fail-sgjqb","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:22:03.575Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"a45e2bcc-66e0-4d8f-a064-35b184b64a04","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:22:03.575Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"a45e2bcc-66e0-4d8f-a064-35b184b64a04","name":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:22:03.575Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"a45e2bcc-66e0-4d8f-a064-35b184b64a04","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:22:03.575Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"a45e2bcc-66e0-4d8f-a064-35b184b64a04"}
{"level":"info","ts":"2026-11-03T15:22:03.575Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"a45e2bcc-66e0-4d8f-a064-35b184b64a04","rayCluster":"fail-sgjqb","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:22:06.581Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"605f1900-f558-412c-baa2-9b3163094a4a","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:22:06.582Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"605f1900-f558-412c-baa2-9b3163094a4a","name":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:22:06.582Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"605f1900-f558-412c-baa2-9b3163094a4a","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:22:06.582Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"605f1900-f558-412c-baa2-9b3163094a4a"}
{"level":"info","ts":"2026-11-03T15:22:06.582Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"605f1900-f558-412c-baa2-9b3163094a4a","rayCluster":"fail-sgjqb","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:22:06.588Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"605f1900-f558-412c-baa2-9b3163094a4a","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","dashboardURL":"fail-sgjqb-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:21:46Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:21:42Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:22:06Z"},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","dashboardURL":"fail-sgjqb-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:21:57Z","stateTransitionTimes":{"ready":"2026-11-03T15:21:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:21:42Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:21:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:22:06.588Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"605f1900-f558-412c-baa2-9b3163094a4a","old JobStatus":"","new JobStatus":"PENDING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-11-03T15:22:06.686Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"975c40fc-f829-44ab-9304-58b6accfd4f5","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:22:06.686Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"975c40fc-f829-44ab-9304-58b6accfd4f5","name":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:22:06.686Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"975c40fc-f829-44ab-9304-58b6accfd4f5","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:22:06.686Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"975c40fc-f829-44ab-9304-58b6accfd4f5"}
{"level":"info","ts":"2026-11-03T15:22:06.686Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"975c40fc-f829-44ab-9304-58b6accfd4f5","rayCluster":"fail-sgjqb","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:22:06.689Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"975c40fc-f829-44ab-9304-58b6accfd4f5","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:22:06Z"},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","dashboardURL":"fail-sgjqb-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:21:57Z","stateTransitionTimes":{"ready":"2026-11-03T15:21:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:21:42Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:21:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:22:06Z"},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","dashboardURL":"fail-sgjqb-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:21:57Z","stateTransitionTimes":{"ready":"2026-11-03T15:21:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:21:42Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:21:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:22:09.689Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"1012c51e-e223-40d8-b885-439097a973b7","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:22:09.689Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"1012c51e-e223-40d8-b885-439097a973b7","name":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:22:09.689Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"1012c51e-e223-40d8-b885-439097a973b7","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:22:09.690Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"1012c51e-e223-40d8-b885-439097a973b7"}
{"level":"info","ts":"2026-11-03T15:22:09.690Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"1012c51e-e223-40d8-b885-439097a973b7","rayCluster":"fail-sgjqb","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:22:09.693Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"1012c51e-e223-40d8-b885-439097a973b7","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:22:06Z"},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","dashboardURL":"fail-sgjqb-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:21:57Z","stateTransitionTimes":{"ready":"2026-11-03T15:21:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:21:42Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:21:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:22:06Z","endTime":"2026-11-03T15:22:07Z"},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","dashboardURL":"fail-sgjqb-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-11-03 15:22:06,121\tINFO job_manager.py:583 -- Runtime env is setting up.\nRunning entrypoint for job fail-z4fr6: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:21:57Z","stateTransitionTimes":{"ready":"2026-11-03T15:21:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:21:42Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:21:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:22:09.693Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"1012c51e-e223-40d8-b885-439097a973b7","old JobStatus":"PENDING","new JobStatus":"FAILED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-11-03T15:22:09.803Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"308a3fd2-9f73-421a-8927-cc010ad31f7e","JobStatus":"FAILED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:22:09.803Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"308a3fd2-9f73-421a-8927-cc010ad31f7e","name":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:22:09.803Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"308a3fd2-9f73-421a-8927-cc010ad31f7e","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:22:09.803Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"308a3fd2-9f73-421a-8927-cc010ad31f7e"}
{"level":"info","ts":"2026-11-03T15:22:09.803Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"308a3fd2-9f73-421a-8927-cc010ad31f7e","rayCluster":"fail-sgjqb","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:22:09.806Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"308a3fd2-9f73-421a-8927-cc010ad31f7e","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:22:06Z","endTime":"2026-11-03T15:22:07Z"},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","dashboardURL":"fail-sgjqb-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-11-03 15:22:06,121\tINFO job_manager.py:583 -- Runtime env is setting up.\nRunning entrypoint for job fail-z4fr6: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:21:57Z","stateTransitionTimes":{"ready":"2026-11-03T15:21:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:21:42Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:21:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:22:06Z","endTime":"2026-11-03T15:22:07Z"},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","dashboardURL":"fail-sgjqb-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-11-03 15:22:06,121\tINFO job_manager.py:583 -- Runtime env is setting up.\nRunning entrypoint for job fail-z4fr6: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:21:57Z","stateTransitionTimes":{"ready":"2026-11-03T15:21:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:21:42Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:21:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:22:12.807Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"23901843-c9dc-4bfa-8d4d-405becc0d746","JobStatus":"FAILED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:22:12.807Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"23901843-c9dc-4bfa-8d4d-405becc0d746","name":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:22:12.807Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"23901843-c9dc-4bfa-8d4d-405becc0d746","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:22:12.807Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"23901843-c9dc-4bfa-8d4d-405becc0d746"}
{"level":"info","ts":"2026-11-03T15:22:12.807Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"23901843-c9dc-4bfa-8d4d-405becc0d746","rayCluster":"fail-sgjqb","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:22:12.810Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"23901843-c9dc-4bfa-8d4d-405becc0d746","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:22:06Z","endTime":"2026-11-03T15:22:07Z"},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","dashboardURL":"fail-sgjqb-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-11-03 15:22:06,121\tINFO job_manager.py:583 -- Runtime env is setting up.\nRunning entrypoint for job fail-z4fr6: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:21:57Z","stateTransitionTimes":{"ready":"2026-11-03T15:21:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:21:42Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:21:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:22:06Z","endTime":"2026-11-03T15:22:07Z"},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","dashboardURL":"fail-sgjqb-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-11-03 15:22:06,121\tINFO job_manager.py:583 -- Runtime env is setting up.\nRunning entrypoint for job fail-z4fr6: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:21:57Z","stateTransitionTimes":{"ready":"2026-11-03T15:21:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:21:42Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:21:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:22:15.811Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"9cac85bb-c141-402d-ad07-25d87cef944c","JobStatus":"FAILED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:22:15.811Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"9cac85bb-c141-402d-ad07-25d87cef944c","name":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:22:15.811Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"9cac85bb-c141-402d-ad07-25d87cef944c","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:22:15.812Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"9cac85bb-c141-402d-ad07-25d87cef944c"}
{"level":"info","ts":"2026-11-03T15:22:15.812Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"9cac85bb-c141-402d-ad07-25d87cef944c","rayCluster":"fail-sgjqb","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:22:15.815Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"9cac85bb-c141-402d-ad07-25d87cef944c","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:22:06Z","endTime":"2026-11-03T15:22:07Z"},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","dashboardURL":"fail-sgjqb-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-11-03 15:22:06,121\tINFO job_manager.py:583 -- Runtime env is setting up.\nRunning entrypoint for job fail-z4fr6: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:21:57Z","stateTransitionTimes":{"ready":"2026-11-03T15:21:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:21:42Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:21:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:22:06Z","endTime":"2026-11-03T15:22:07Z"},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","dashboardURL":"fail-sgjqb-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-11-03 15:22:06,121\tINFO job_manager.py:583 -- Runtime env is setting up.\nRunning entrypoint for job fail-z4fr6: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:21:57Z","stateTransitionTimes":{"ready":"2026-11-03T15:21:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:21:42Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:21:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:22:17.639Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"63f3939c-3fe3-4504-8d38-d5c498d3e38b","JobStatus":"FAILED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:22:17.639Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"63f3939c-3fe3-4504-8d38-d5c498d3e38b","name":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:22:17.639Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"63f3939c-3fe3-4504-8d38-d5c498d3e38b","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:22:17.640Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"63f3939c-3fe3-4504-8d38-d5c498d3e38b"}
{"level":"info","ts":"2026-11-03T15:22:17.640Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"63f3939c-3fe3-4504-8d38-d5c498d3e38b","rayCluster":"fail-sgjqb","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:22:17.643Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"63f3939c-3fe3-4504-8d38-d5c498d3e38b","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:22:06Z","endTime":"2026-11-03T15:22:07Z"},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","dashboardURL":"fail-sgjqb-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-11-03 15:22:06,121\tINFO job_manager.py:583 -- Runtime env is setting up.\nRunning entrypoint for job fail-z4fr6: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:21:57Z","stateTransitionTimes":{"ready":"2026-11-03T15:21:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:21:42Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:21:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:22:06Z","endTime":"2026-11-03T15:22:07Z"},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","dashboardURL":"fail-sgjqb-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-11-03 15:22:06,121\tINFO job_manager.py:583 -- Runtime env is setting up.\nRunning entrypoint for job fail-z4fr6: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:21:57Z","stateTransitionTimes":{"ready":"2026-11-03T15:21:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:21:42Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:21:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:22:18.649Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"fc15999c-27c9-4a95-a888-bf7fa974df41","JobStatus":"FAILED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:22:18.649Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"fc15999c-27c9-4a95-a888-bf7fa974df41","name":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:22:18.649Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"fc15999c-27c9-4a95-a888-bf7fa974df41","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:22:18.649Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"fc15999c-27c9-4a95-a888-bf7fa974df41"}
{"level":"info","ts":"2026-11-03T15:22:18.649Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"fc15999c-27c9-4a95-a888-bf7fa974df41","rayCluster":"fail-sgjqb","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:22:18.652Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"fc15999c-27c9-4a95-a888-bf7fa974df41","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:22:06Z","endTime":"2026-11-03T15:22:07Z"},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","dashboardURL":"fail-sgjqb-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-11-03 15:22:06,121\tINFO job_manager.py:583 -- Runtime env is setting up.\nRunning entrypoint for job fail-z4fr6: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:21:57Z","stateTransitionTimes":{"ready":"2026-11-03T15:21:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:21:42Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:21:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:22:06Z","endTime":"2026-11-03T15:22:07Z"},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","dashboardURL":"fail-sgjqb-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-11-03 15:22:06,121\tINFO job_manager.py:583 -- Runtime env is setting up.\nRunning entrypoint for job fail-z4fr6: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:21:57Z","stateTransitionTimes":{"ready":"2026-11-03T15:21:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:21:42Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:21:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:22:18.668Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"d242acd5-4e3a-4493-bcbf-f18a3499bc04","JobStatus":"FAILED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:22:18.668Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"d242acd5-4e3a-4493-bcbf-f18a3499bc04","name":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:22:18.668Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"d242acd5-4e3a-4493-bcbf-f18a3499bc04","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:22:18.668Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"d242acd5-4e3a-4493-bcbf-f18a3499bc04"}
{"level":"info","ts":"2026-11-03T15:22:18.668Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"d242acd5-4e3a-4493-bcbf-f18a3499bc04","rayCluster":"fail-sgjqb","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:22:18.681Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"d242acd5-4e3a-4493-bcbf-f18a3499bc04","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:22:06Z","endTime":"2026-11-03T15:22:07Z"},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","dashboardURL":"fail-sgjqb-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Running","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-11-03 15:22:06,121\tINFO job_manager.py:583 -- Runtime env is setting up.\nRunning entrypoint for job fail-z4fr6: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:21:57Z","stateTransitionTimes":{"ready":"2026-11-03T15:21:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:21:42Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:21:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:22:06Z","endTime":"2026-11-03T15:22:07Z"},"jobId":"fail-z4fr6","rayClusterName":"fail-sgjqb","dashboardURL":"fail-sgjqb-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Failed","reason":"AppFailed","message":"Job entrypoint command failed with exit code 1, last available logs (truncated to 20,000 chars):\n2026-11-03 15:22:06,121\tINFO job_manager.py:583 -- Runtime env is setting up.\nRunning entrypoint for job fail-z4fr6: python /home/ray/jobs/fail.py\nSomething is seriously wrong.\n","startTime":"2026-11-03T15:20:04Z","succeeded":0,"failed":1,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:21:57Z","stateTransitionTimes":{"ready":"2026-11-03T15:21:57Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.39","serviceIP":"10.132.0.39","podName":"fail-sgjqb-head-8pdmr","serviceName":"fail-sgjqb-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:21:42Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:21:57Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:20:05Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:20:05Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:22:18.681Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"d242acd5-4e3a-4493-bcbf-f18a3499bc04","old JobStatus":"FAILED","new JobStatus":"FAILED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Failed"}
{"level":"info","ts":"2026-11-03T15:22:18.791Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"773eca89-e125-4c53-b1c5-26213f65a639","JobStatus":"FAILED","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:22:19.270Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"2c78c8fe-0664-41ca-a3f5-a5951aafe525","DeletionTimestamp":"2026-11-03 15:22:19 +0000 UTC"}
{"level":"info","ts":"2026-11-03T15:22:19.270Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"2c78c8fe-0664-41ca-a3f5-a5951aafe525","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-11-03T15:22:19.334Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"077981dd-ee87-4165-904a-1e6709b3a217"}
{"level":"info","ts":"2026-11-03T15:22:19.367Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"e73ee59e-f282-45e8-a6a0-d72a45d69dff"}
{"level":"info","ts":"2026-11-03T15:22:19.381Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"947e9ccf-b3c4-4d61-8599-6929e0849b1c"}
{"level":"info","ts":"2026-11-03T15:22:19.382Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"reconcileID":"b674d827-bed2-492d-b6ca-7dac20c32124"}
{"level":"info","ts":"2026-11-03T15:22:19.382Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"6edc36f1-8af9-4b0b-8aba-3e53c3603b7e"}
{"level":"info","ts":"2026-11-03T15:22:19.382Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"fail-sgjqb","reconcileID":"f8c2bc8a-9531-4f3a-b458-b824a84a8a5b","namespacedName":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:22:19.382Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"fail-sgjqb","reconcileID":"f8c2bc8a-9531-4f3a-b458-b824a84a8a5b","cluster":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:22:19.382Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"fail-sgjqb","reconcileID":"f8c2bc8a-9531-4f3a-b458-b824a84a8a5b","cluster":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:22:19.382Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"fail-sgjqb","reconcileID":"f8c2bc8a-9531-4f3a-b458-b824a84a8a5b","namespace":"test-ns-kbx4w","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-11-03T15:22:19.435Z","logger":"authentication-controller","msg":"Deleted shared ReferenceGrant (last cluster with auth)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"fail-sgjqb","reconcileID":"f8c2bc8a-9531-4f3a-b458-b824a84a8a5b","name":"kuberay-gateway-access","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:22:19.488Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"fail-sgjqb","reconcileID":"f8c2bc8a-9531-4f3a-b458-b824a84a8a5b","name":"test-ns-kbx4w-fail-sgjqb","namespace":"opendatahub"}
{"level":"info","ts":"2026-11-03T15:22:19.541Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"fail-sgjqb","reconcileID":"f8c2bc8a-9531-4f3a-b458-b824a84a8a5b","configMap":"kube-rbac-proxy-config-fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:22:19.593Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"fail-sgjqb","reconcileID":"f8c2bc8a-9531-4f3a-b458-b824a84a8a5b","serviceAccount":"fail-sgjqb-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-11-03T15:22:19.763Z","logger":"raycluster-resource","msg":"default","name":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:22:19.763Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"fail-sgjqb","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:22:19.940Z","logger":"raycluster-resource","msg":"validate update","name":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:22:19.999Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"fail-sgjqb","reconcileID":"f8c2bc8a-9531-4f3a-b458-b824a84a8a5b","cluster":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:22:19.999Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"fail-sgjqb","reconcileID":"66ea86f7-6852-4766-aee5-841abcf13a2a","namespacedName":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:22:19.999Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"fail-sgjqb","reconcileID":"66ea86f7-6852-4766-aee5-841abcf13a2a","cluster":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:22:19.999Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"fail-sgjqb","reconcileID":"66ea86f7-6852-4766-aee5-841abcf13a2a","cluster":"fail-sgjqb"}
{"level":"info","ts":"2026-11-03T15:22:19.999Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"fail-sgjqb","reconcileID":"66ea86f7-6852-4766-aee5-841abcf13a2a","namespace":"test-ns-kbx4w","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-11-03T15:22:19.999Z","logger":"authentication-controller","msg":"ReferenceGrant already deleted","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"fail-sgjqb","reconcileID":"66ea86f7-6852-4766-aee5-841abcf13a2a","name":"kuberay-gateway-access"}
{"level":"info","ts":"2026-11-03T15:22:19.999Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"fail","namespace":"test-ns-kbx4w"},"reconcileID":"78fa5200-9a08-4785-b538-effce8162c00"}
{"level":"error","ts":"2026-11-03T15:22:20.095Z","logger":"authentication-controller","msg":"Failed to remove finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"fail-sgjqb","reconcileID":"66ea86f7-6852-4766-aee5-841abcf13a2a","error":"rayclusters.ray.io \"fail-sgjqb\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleDeletion\n\t/workspace/controllers/ray/authentication_controller.go:195\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:22:20.095Z","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"fail-sgjqb","reconcileID":"66ea86f7-6852-4766-aee5-841abcf13a2a"}
{"level":"error","ts":"2026-11-03T15:22:20.095Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"fail-sgjqb","reconcileID":"66ea86f7-6852-4766-aee5-841abcf13a2a","error":"rayclusters.ray.io \"fail-sgjqb\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:22:20.095Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"fail-sgjqb","reconcileID":"ee876aa2-0fa5-43fd-bae7-0599eb93ccc9","namespacedName":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:22:20.095Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"fail-sgjqb","reconcileID":"ee876aa2-0fa5-43fd-bae7-0599eb93ccc9"}
{"level":"info","ts":"2026-11-03T15:22:20.095Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"fail-sgjqb","reconcileID":"ee876aa2-0fa5-43fd-bae7-0599eb93ccc9","namespace":"test-ns-kbx4w","authClusters":0}
{"level":"info","ts":"2026-11-03T15:22:20.095Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-sgjqb","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"fail-sgjqb","reconcileID":"ee876aa2-0fa5-43fd-bae7-0599eb93ccc9","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:22:20.347Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"0bc1f204-8409-4482-ab27-62495cbc993b","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:22:20.347Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"0bc1f204-8409-4482-ab27-62495cbc993b","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-11-03T15:22:20.407Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"0bc1f204-8409-4482-ab27-62495cbc993b"}
{"level":"info","ts":"2026-11-03T15:22:20.407Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"0bc1f204-8409-4482-ab27-62495cbc993b","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-11-03T15:22:20.407Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"0bc1f204-8409-4482-ab27-62495cbc993b","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:22:20.407Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"0bc1f204-8409-4482-ab27-62495cbc993b","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:22:20.468Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"65f99685-8d2a-4b28-9e0d-eb772d5f1fdc","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:22:20.468Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"65f99685-8d2a-4b28-9e0d-eb772d5f1fdc","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:22:20.468Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"65f99685-8d2a-4b28-9e0d-eb772d5f1fdc","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:22:20.468Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"65f99685-8d2a-4b28-9e0d-eb772d5f1fdc","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:22:20.519Z","logger":"raycluster-resource","msg":"default","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:22:20.519Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:22:20.574Z","logger":"raycluster-resource","msg":"validate create","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:22:20.633Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"ca770355-34ee-4f22-9999-929ee9a5ec80"}
{"level":"info","ts":"2026-11-03T15:22:20.633Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"ca770355-34ee-4f22-9999-929ee9a5ec80"}
{"level":"info","ts":"2026-11-03T15:22:20.633Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"72afc275-9b43-4498-b79d-c139630e61f3","cluster":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:22:20.633Z","logger":"controllers.NetworkPolicy","msg":"Reconciling NetworkPolicies for RayCluster","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"c8fb6ce9-ad9a-4bc6-843b-f68117f35f37","cluster":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:22:20.634Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"fail-k8s-job-g5dmq","reconcileID":"765f5027-d067-4cc0-9f9e-6273eba47f31","namespacedName":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:22:20.634Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"fail-k8s-job-g5dmq","reconcileID":"765f5027-d067-4cc0-9f9e-6273eba47f31","mode":"IntegratedOAuth","cluster":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:22:20.634Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"fail-k8s-job-g5dmq","reconcileID":"765f5027-d067-4cc0-9f9e-6273eba47f31","cluster":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:22:20.634Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"fail-k8s-job-g5dmq","reconcileID":"765f5027-d067-4cc0-9f9e-6273eba47f31","cluster":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:22:20.634Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"65f99685-8d2a-4b28-9e0d-eb772d5f1fdc","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:22:20.634Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"65f99685-8d2a-4b28-9e0d-eb772d5f1fdc"}
{"level":"info","ts":"2026-11-03T15:22:20.634Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"65f99685-8d2a-4b28-9e0d-eb772d5f1fdc","RayCluster":"fail-k8s-job-g5dmq","State":""}
{"level":"info","ts":"2026-11-03T15:22:20.634Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"65f99685-8d2a-4b28-9e0d-eb772d5f1fdc","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:22:20.634Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"f79a0d04-3fe6-43f7-899d-cde005fcb52e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:22:20.634Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"f79a0d04-3fe6-43f7-899d-cde005fcb52e","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:22:20.634Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"f79a0d04-3fe6-43f7-899d-cde005fcb52e","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:22:20.634Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"f79a0d04-3fe6-43f7-899d-cde005fcb52e"}
{"level":"info","ts":"2026-11-03T15:22:20.634Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"f79a0d04-3fe6-43f7-899d-cde005fcb52e","RayCluster":"fail-k8s-job-g5dmq","State":""}
{"level":"info","ts":"2026-11-03T15:22:20.634Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"f79a0d04-3fe6-43f7-899d-cde005fcb52e","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:22:20.686Z","logger":"raycluster-resource","msg":"default","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:22:20.686Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:22:20.689Z","logger":"controllers.NetworkPolicy","msg":"Successfully created NetworkPolicy","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"c8fb6ce9-ad9a-4bc6-843b-f68117f35f37","name":"fail-k8s-job-g5dmq-head"}
{"level":"info","ts":"2026-11-03T15:22:20.689Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"ca770355-34ee-4f22-9999-929ee9a5ec80","name":"fail-k8s-job-g5dmq-head-svc"}
{"level":"info","ts":"2026-11-03T15:22:20.689Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"ca770355-34ee-4f22-9999-929ee9a5ec80"}
{"level":"info","ts":"2026-11-03T15:22:20.740Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"ca770355-34ee-4f22-9999-929ee9a5ec80","error":"certificate ray-head-cert-fail-k8s-job-g5dmq not found: certificates.cert-manager.io \"ray-head-cert-fail-k8s-job-g5dmq\" not found"}
{"level":"error","ts":"2026-11-03T15:22:20.740Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"ca770355-34ee-4f22-9999-929ee9a5ec80","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-fail-k8s-job-g5dmq not found: certificates.cert-manager.io \"ray-head-cert-fail-k8s-job-g5dmq\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:22:20.740Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"ca770355-34ee-4f22-9999-929ee9a5ec80","error":"mTLS secrets not ready: certificate ray-head-cert-fail-k8s-job-g5dmq not found: certificates.cert-manager.io \"ray-head-cert-fail-k8s-job-g5dmq\" not found"}
{"level":"info","ts":"2026-11-03T15:22:20.740Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"ca770355-34ee-4f22-9999-929ee9a5ec80","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"fail-k8s-job-g5dmq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:22:20.740Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"ca770355-34ee-4f22-9999-929ee9a5ec80","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"fail-k8s-job-g5dmq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:22:20.740Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"ca770355-34ee-4f22-9999-929ee9a5ec80","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"fail-k8s-job-g5dmq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:22:20.740Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"ca770355-34ee-4f22-9999-929ee9a5ec80","name":"fail-k8s-job-g5dmq","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:22:20.742Z","logger":"controllers.NetworkPolicy","msg":"Successfully created NetworkPolicy","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"c8fb6ce9-ad9a-4bc6-843b-f68117f35f37","name":"fail-k8s-job-g5dmq-workers-small-group"}
{"level":"info","ts":"2026-11-03T15:22:20.742Z","logger":"controllers.NetworkPolicy","msg":"Successfully reconciled NetworkPolicies for RayCluster","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"c8fb6ce9-ad9a-4bc6-843b-f68117f35f37","cluster":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:22:20.742Z","logger":"controllers.NetworkPolicy","msg":"Reconciling NetworkPolicies for RayCluster","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"11122951-2c60-46a7-90a4-f89f23e77a06","cluster":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:22:20.742Z","logger":"raycluster-resource","msg":"validate update","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:22:20.802Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"ca770355-34ee-4f22-9999-929ee9a5ec80"}
{"level":"error","ts":"2026-11-03T15:22:20.802Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"ca770355-34ee-4f22-9999-929ee9a5ec80","error":"mTLS secrets not ready: certificate ray-head-cert-fail-k8s-job-g5dmq not found: certificates.cert-manager.io \"ray-head-cert-fail-k8s-job-g5dmq\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:22:20.802Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"cf16d94a-a0e7-45bf-9ff0-b75e83312088"}
{"level":"info","ts":"2026-11-03T15:22:20.802Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"cf16d94a-a0e7-45bf-9ff0-b75e83312088"}
{"level":"info","ts":"2026-11-03T15:22:20.802Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"cf16d94a-a0e7-45bf-9ff0-b75e83312088","1 head service found":"fail-k8s-job-g5dmq-head-svc"}
{"level":"info","ts":"2026-11-03T15:22:20.802Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"cf16d94a-a0e7-45bf-9ff0-b75e83312088"}
{"level":"info","ts":"2026-11-03T15:22:20.802Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"d23cd6f2-2676-40df-9957-1e6f5eab0d77","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:22:20.802Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"d23cd6f2-2676-40df-9957-1e6f5eab0d77","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:22:20.803Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"d23cd6f2-2676-40df-9957-1e6f5eab0d77","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:22:20.803Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"d23cd6f2-2676-40df-9957-1e6f5eab0d77"}
{"level":"info","ts":"2026-11-03T15:22:20.803Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"d23cd6f2-2676-40df-9957-1e6f5eab0d77","RayCluster":"fail-k8s-job-g5dmq","State":""}
{"level":"info","ts":"2026-11-03T15:22:20.803Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"d23cd6f2-2676-40df-9957-1e6f5eab0d77","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:22:20.803Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"d23cd6f2-2676-40df-9957-1e6f5eab0d77","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:22:20.804Z","logger":"raycluster-resource","msg":"default","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:22:20.804Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:22:20.852Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"cf16d94a-a0e7-45bf-9ff0-b75e83312088","error":"certificate ray-head-cert-fail-k8s-job-g5dmq not found: certificates.cert-manager.io \"ray-head-cert-fail-k8s-job-g5dmq\" not found"}
{"level":"error","ts":"2026-11-03T15:22:20.852Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"cf16d94a-a0e7-45bf-9ff0-b75e83312088","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-fail-k8s-job-g5dmq not found: certificates.cert-manager.io \"ray-head-cert-fail-k8s-job-g5dmq\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:22:20.852Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"cf16d94a-a0e7-45bf-9ff0-b75e83312088","error":"mTLS secrets not ready: certificate ray-head-cert-fail-k8s-job-g5dmq not found: certificates.cert-manager.io \"ray-head-cert-fail-k8s-job-g5dmq\" not found"}
{"level":"info","ts":"2026-11-03T15:22:20.852Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"cf16d94a-a0e7-45bf-9ff0-b75e83312088","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"fail-k8s-job-g5dmq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:22:20.852Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"cf16d94a-a0e7-45bf-9ff0-b75e83312088","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"fail-k8s-job-g5dmq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:22:20.852Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"cf16d94a-a0e7-45bf-9ff0-b75e83312088","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"fail-k8s-job-g5dmq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:22:20.852Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"cf16d94a-a0e7-45bf-9ff0-b75e83312088","name":"fail-k8s-job-g5dmq","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"error","ts":"2026-11-03T15:22:20.853Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"fail-k8s-job-g5dmq","reconcileID":"765f5027-d067-4cc0-9f9e-6273eba47f31","error":"failed to add finalizer: Operation cannot be fulfilled on rayclusters.ray.io \"fail-k8s-job-g5dmq\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:167\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:22:20.853Z","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"fail-k8s-job-g5dmq","reconcileID":"765f5027-d067-4cc0-9f9e-6273eba47f31"}
{"level":"error","ts":"2026-11-03T15:22:20.853Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"fail-k8s-job-g5dmq","reconcileID":"765f5027-d067-4cc0-9f9e-6273eba47f31","error":"failed to add finalizer: Operation cannot be fulfilled on rayclusters.ray.io \"fail-k8s-job-g5dmq\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:22:20.853Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"fail-k8s-job-g5dmq","reconcileID":"975753b9-d24c-4b6f-8213-78dfc9dd1cbd","namespacedName":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:22:20.853Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"fail-k8s-job-g5dmq","reconcileID":"975753b9-d24c-4b6f-8213-78dfc9dd1cbd","mode":"IntegratedOAuth","cluster":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:22:20.853Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"fail-k8s-job-g5dmq","reconcileID":"975753b9-d24c-4b6f-8213-78dfc9dd1cbd","cluster":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:22:20.853Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"fail-k8s-job-g5dmq","reconcileID":"975753b9-d24c-4b6f-8213-78dfc9dd1cbd","cluster":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:22:20.857Z","logger":"controllers.NetworkPolicy","msg":"Successfully reconciled NetworkPolicies for RayCluster","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"11122951-2c60-46a7-90a4-f89f23e77a06","cluster":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:22:20.864Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"57fcaaa9-78ae-4738-953a-3a441712c500","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:22:20.864Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"57fcaaa9-78ae-4738-953a-3a441712c500","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:22:20.864Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"57fcaaa9-78ae-4738-953a-3a441712c500","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:22:20.864Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"57fcaaa9-78ae-4738-953a-3a441712c500"}
{"level":"info","ts":"2026-11-03T15:22:20.864Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"57fcaaa9-78ae-4738-953a-3a441712c500","RayCluster":"fail-k8s-job-g5dmq","State":""}
{"level":"info","ts":"2026-11-03T15:22:20.864Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"57fcaaa9-78ae-4738-953a-3a441712c500","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:22:20.906Z","logger":"raycluster-resource","msg":"default","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:22:20.906Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:22:20.907Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"cf16d94a-a0e7-45bf-9ff0-b75e83312088","name":"fail-k8s-job-g5dmq","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-k8s-job-g5dmq\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-kbx4w","name":"fail-k8s-job-g5dmq"}}
{"level":"info","ts":"2026-11-03T15:22:20.907Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"cf16d94a-a0e7-45bf-9ff0-b75e83312088"}
{"level":"error","ts":"2026-11-03T15:22:20.907Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"cf16d94a-a0e7-45bf-9ff0-b75e83312088","error":"mTLS secrets not ready: certificate ray-head-cert-fail-k8s-job-g5dmq not found: certificates.cert-manager.io \"ray-head-cert-fail-k8s-job-g5dmq\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:22:20.918Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"753076f5-dfef-44b7-830e-c0fabb0bc6d9"}
{"level":"info","ts":"2026-11-03T15:22:20.918Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"753076f5-dfef-44b7-830e-c0fabb0bc6d9"}
{"level":"info","ts":"2026-11-03T15:22:20.918Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"753076f5-dfef-44b7-830e-c0fabb0bc6d9","1 head service found":"fail-k8s-job-g5dmq-head-svc"}
{"level":"info","ts":"2026-11-03T15:22:20.918Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"753076f5-dfef-44b7-830e-c0fabb0bc6d9"}
{"level":"info","ts":"2026-11-03T15:22:20.959Z","logger":"raycluster-resource","msg":"validate update","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:22:20.969Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"753076f5-dfef-44b7-830e-c0fabb0bc6d9","error":"certificate ray-head-cert-fail-k8s-job-g5dmq not found: certificates.cert-manager.io \"ray-head-cert-fail-k8s-job-g5dmq\" not found"}
{"level":"error","ts":"2026-11-03T15:22:20.969Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"753076f5-dfef-44b7-830e-c0fabb0bc6d9","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-fail-k8s-job-g5dmq not found: certificates.cert-manager.io \"ray-head-cert-fail-k8s-job-g5dmq\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:22:20.969Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"753076f5-dfef-44b7-830e-c0fabb0bc6d9","error":"mTLS secrets not ready: certificate ray-head-cert-fail-k8s-job-g5dmq not found: certificates.cert-manager.io \"ray-head-cert-fail-k8s-job-g5dmq\" not found"}
{"level":"info","ts":"2026-11-03T15:22:20.969Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"753076f5-dfef-44b7-830e-c0fabb0bc6d9","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"fail-k8s-job-g5dmq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:22:20.969Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"753076f5-dfef-44b7-830e-c0fabb0bc6d9","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"fail-k8s-job-g5dmq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:22:20.969Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"753076f5-dfef-44b7-830e-c0fabb0bc6d9","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"fail-k8s-job-g5dmq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:22:20.969Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"753076f5-dfef-44b7-830e-c0fabb0bc6d9"}
{"level":"error","ts":"2026-11-03T15:22:20.969Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"753076f5-dfef-44b7-830e-c0fabb0bc6d9","error":"mTLS secrets not ready: certificate ray-head-cert-fail-k8s-job-g5dmq not found: certificates.cert-manager.io \"ray-head-cert-fail-k8s-job-g5dmq\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:22:20.989Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"1a7cd866-300f-441b-a284-a218c9084111"}
{"level":"info","ts":"2026-11-03T15:22:20.989Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"1a7cd866-300f-441b-a284-a218c9084111"}
{"level":"info","ts":"2026-11-03T15:22:20.989Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"1a7cd866-300f-441b-a284-a218c9084111","1 head service found":"fail-k8s-job-g5dmq-head-svc"}
{"level":"info","ts":"2026-11-03T15:22:20.989Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"1a7cd866-300f-441b-a284-a218c9084111"}
{"level":"info","ts":"2026-11-03T15:22:21.019Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"5b0696ee-ac55-473d-94f3-4cfe7f4fb038","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:22:21.019Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"5b0696ee-ac55-473d-94f3-4cfe7f4fb038","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:22:21.019Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"5b0696ee-ac55-473d-94f3-4cfe7f4fb038","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:22:21.019Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"5b0696ee-ac55-473d-94f3-4cfe7f4fb038"}
{"level":"info","ts":"2026-11-03T15:22:21.019Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"5b0696ee-ac55-473d-94f3-4cfe7f4fb038","RayCluster":"fail-k8s-job-g5dmq","State":""}
{"level":"info","ts":"2026-11-03T15:22:21.019Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"5b0696ee-ac55-473d-94f3-4cfe7f4fb038","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:22:21.039Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"1a7cd866-300f-441b-a284-a218c9084111","error":"certificate ray-head-cert-fail-k8s-job-g5dmq not found: certificates.cert-manager.io \"ray-head-cert-fail-k8s-job-g5dmq\" not found"}
{"level":"error","ts":"2026-11-03T15:22:21.039Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"1a7cd866-300f-441b-a284-a218c9084111","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-fail-k8s-job-g5dmq not found: certificates.cert-manager.io \"ray-head-cert-fail-k8s-job-g5dmq\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:22:21.039Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"1a7cd866-300f-441b-a284-a218c9084111","error":"mTLS secrets not ready: certificate ray-head-cert-fail-k8s-job-g5dmq not found: certificates.cert-manager.io \"ray-head-cert-fail-k8s-job-g5dmq\" not found"}
{"level":"info","ts":"2026-11-03T15:22:21.039Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"1a7cd866-300f-441b-a284-a218c9084111","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"fail-k8s-job-g5dmq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:22:21.039Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"1a7cd866-300f-441b-a284-a218c9084111","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"fail-k8s-job-g5dmq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:22:21.039Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"1a7cd866-300f-441b-a284-a218c9084111","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"fail-k8s-job-g5dmq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:22:21.039Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"1a7cd866-300f-441b-a284-a218c9084111"}
{"level":"error","ts":"2026-11-03T15:22:21.039Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"1a7cd866-300f-441b-a284-a218c9084111","error":"mTLS secrets not ready: certificate ray-head-cert-fail-k8s-job-g5dmq not found: certificates.cert-manager.io \"ray-head-cert-fail-k8s-job-g5dmq\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:22:21.071Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"fail-k8s-job-g5dmq","reconcileID":"975753b9-d24c-4b6f-8213-78dfc9dd1cbd","name":"fail-k8s-job-g5dmq-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-11-03T15:22:21.080Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"00c59317-80a5-4fbf-8d16-c4502bf8ba8c"}
{"level":"info","ts":"2026-11-03T15:22:21.080Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"00c59317-80a5-4fbf-8d16-c4502bf8ba8c"}
{"level":"info","ts":"2026-11-03T15:22:21.080Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"00c59317-80a5-4fbf-8d16-c4502bf8ba8c","1 head service found":"fail-k8s-job-g5dmq-head-svc"}
{"level":"info","ts":"2026-11-03T15:22:21.080Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"00c59317-80a5-4fbf-8d16-c4502bf8ba8c"}
{"level":"info","ts":"2026-11-03T15:22:21.125Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"fail-k8s-job-g5dmq","reconcileID":"975753b9-d24c-4b6f-8213-78dfc9dd1cbd","name":"kuberay-gateway-access","namespace":"test-ns-kbx4w","operation":"created"}
{"level":"info","ts":"2026-11-03T15:22:21.130Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"00c59317-80a5-4fbf-8d16-c4502bf8ba8c","error":"certificate ray-head-cert-fail-k8s-job-g5dmq not found: certificates.cert-manager.io \"ray-head-cert-fail-k8s-job-g5dmq\" not found"}
{"level":"error","ts":"2026-11-03T15:22:21.130Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"00c59317-80a5-4fbf-8d16-c4502bf8ba8c","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-fail-k8s-job-g5dmq not found: certificates.cert-manager.io \"ray-head-cert-fail-k8s-job-g5dmq\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:22:21.130Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"00c59317-80a5-4fbf-8d16-c4502bf8ba8c","error":"mTLS secrets not ready: certificate ray-head-cert-fail-k8s-job-g5dmq not found: certificates.cert-manager.io \"ray-head-cert-fail-k8s-job-g5dmq\" not found"}
{"level":"info","ts":"2026-11-03T15:22:21.130Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"00c59317-80a5-4fbf-8d16-c4502bf8ba8c","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"fail-k8s-job-g5dmq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:22:21.130Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"00c59317-80a5-4fbf-8d16-c4502bf8ba8c","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"fail-k8s-job-g5dmq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:22:21.130Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"00c59317-80a5-4fbf-8d16-c4502bf8ba8c","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"fail-k8s-job-g5dmq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:22:21.130Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"00c59317-80a5-4fbf-8d16-c4502bf8ba8c"}
{"level":"error","ts":"2026-11-03T15:22:21.130Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"00c59317-80a5-4fbf-8d16-c4502bf8ba8c","error":"mTLS secrets not ready: certificate ray-head-cert-fail-k8s-job-g5dmq not found: certificates.cert-manager.io \"ray-head-cert-fail-k8s-job-g5dmq\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:22:21.180Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"fail-k8s-job-g5dmq","reconcileID":"975753b9-d24c-4b6f-8213-78dfc9dd1cbd","name":"test-ns-kbx4w-fail-k8s-job-g5dmq","operation":"created"}
{"level":"info","ts":"2026-11-03T15:22:21.210Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"0dc49927-3ace-43cf-8997-852032e83828"}
{"level":"info","ts":"2026-11-03T15:22:21.210Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"0dc49927-3ace-43cf-8997-852032e83828"}
{"level":"info","ts":"2026-11-03T15:22:21.210Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"0dc49927-3ace-43cf-8997-852032e83828","1 head service found":"fail-k8s-job-g5dmq-head-svc"}
{"level":"info","ts":"2026-11-03T15:22:21.210Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"0dc49927-3ace-43cf-8997-852032e83828"}
{"level":"info","ts":"2026-11-03T15:22:21.233Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"fail-k8s-job-g5dmq","reconcileID":"975753b9-d24c-4b6f-8213-78dfc9dd1cbd","name":"kube-rbac-proxy-config-fail-k8s-job-g5dmq","operation":"created"}
{"level":"info","ts":"2026-11-03T15:22:21.260Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"0dc49927-3ace-43cf-8997-852032e83828","error":"certificate ray-head-cert-fail-k8s-job-g5dmq not found: certificates.cert-manager.io \"ray-head-cert-fail-k8s-job-g5dmq\" not found"}
{"level":"error","ts":"2026-11-03T15:22:21.260Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"0dc49927-3ace-43cf-8997-852032e83828","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-fail-k8s-job-g5dmq not found: certificates.cert-manager.io \"ray-head-cert-fail-k8s-job-g5dmq\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:22:21.260Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"0dc49927-3ace-43cf-8997-852032e83828","error":"mTLS secrets not ready: certificate ray-head-cert-fail-k8s-job-g5dmq not found: certificates.cert-manager.io \"ray-head-cert-fail-k8s-job-g5dmq\" not found"}
{"level":"info","ts":"2026-11-03T15:22:21.260Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"0dc49927-3ace-43cf-8997-852032e83828","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"fail-k8s-job-g5dmq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:22:21.260Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"0dc49927-3ace-43cf-8997-852032e83828","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"fail-k8s-job-g5dmq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:22:21.260Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"0dc49927-3ace-43cf-8997-852032e83828","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"fail-k8s-job-g5dmq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:22:21.260Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"0dc49927-3ace-43cf-8997-852032e83828"}
{"level":"error","ts":"2026-11-03T15:22:21.260Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"0dc49927-3ace-43cf-8997-852032e83828","error":"mTLS secrets not ready: certificate ray-head-cert-fail-k8s-job-g5dmq not found: certificates.cert-manager.io \"ray-head-cert-fail-k8s-job-g5dmq\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-03T15:22:21.288Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"fail-k8s-job-g5dmq","reconcileID":"975753b9-d24c-4b6f-8213-78dfc9dd1cbd","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"fail-k8s-job-g5dmq\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:167\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:22:21.288Z","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"fail-k8s-job-g5dmq","reconcileID":"975753b9-d24c-4b6f-8213-78dfc9dd1cbd"}
{"level":"error","ts":"2026-11-03T15:22:21.288Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"fail-k8s-job-g5dmq","reconcileID":"975753b9-d24c-4b6f-8213-78dfc9dd1cbd","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"fail-k8s-job-g5dmq\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:22:21.288Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"fail-k8s-job-g5dmq","reconcileID":"2711b999-abd4-4f3e-b3e9-04af4ce2b923","namespacedName":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:22:21.288Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"fail-k8s-job-g5dmq","reconcileID":"2711b999-abd4-4f3e-b3e9-04af4ce2b923","mode":"IntegratedOAuth","cluster":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:22:21.288Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"fail-k8s-job-g5dmq","reconcileID":"2711b999-abd4-4f3e-b3e9-04af4ce2b923","cluster":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:22:21.349Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"fail-k8s-job-g5dmq","reconcileID":"2711b999-abd4-4f3e-b3e9-04af4ce2b923","cluster":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:22:21.349Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"f38bb0d8-b726-4e63-84e2-3055fa722a9e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:22:21.349Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"f38bb0d8-b726-4e63-84e2-3055fa722a9e","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:22:21.349Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"f38bb0d8-b726-4e63-84e2-3055fa722a9e","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:22:21.350Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"f38bb0d8-b726-4e63-84e2-3055fa722a9e"}
{"level":"info","ts":"2026-11-03T15:22:21.350Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"f38bb0d8-b726-4e63-84e2-3055fa722a9e","RayCluster":"fail-k8s-job-g5dmq","State":""}
{"level":"info","ts":"2026-11-03T15:22:21.350Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"f38bb0d8-b726-4e63-84e2-3055fa722a9e","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:22:21.350Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"f38bb0d8-b726-4e63-84e2-3055fa722a9e","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:22:21.411Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"f63040db-4a51-40d6-9fe4-c82f744745ec","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:22:21.411Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"f63040db-4a51-40d6-9fe4-c82f744745ec","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:22:21.411Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"f63040db-4a51-40d6-9fe4-c82f744745ec","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:22:21.411Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"f63040db-4a51-40d6-9fe4-c82f744745ec"}
{"level":"info","ts":"2026-11-03T15:22:21.411Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"f63040db-4a51-40d6-9fe4-c82f744745ec","RayCluster":"fail-k8s-job-g5dmq","State":""}
{"level":"info","ts":"2026-11-03T15:22:21.411Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"f63040db-4a51-40d6-9fe4-c82f744745ec","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:22:21.421Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"1967bd07-ce60-44c5-b2d0-6d685132e6e8"}
{"level":"info","ts":"2026-11-03T15:22:21.421Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"1967bd07-ce60-44c5-b2d0-6d685132e6e8"}
{"level":"info","ts":"2026-11-03T15:22:21.421Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"1967bd07-ce60-44c5-b2d0-6d685132e6e8","1 head service found":"fail-k8s-job-g5dmq-head-svc"}
{"level":"info","ts":"2026-11-03T15:22:21.421Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"1967bd07-ce60-44c5-b2d0-6d685132e6e8"}
{"level":"info","ts":"2026-11-03T15:22:21.472Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"1967bd07-ce60-44c5-b2d0-6d685132e6e8","error":"certificate ray-head-cert-fail-k8s-job-g5dmq is not ready at current generation 1"}
{"level":"error","ts":"2026-11-03T15:22:21.472Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"1967bd07-ce60-44c5-b2d0-6d685132e6e8","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-fail-k8s-job-g5dmq is not ready at current generation 1","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:22:21.472Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"1967bd07-ce60-44c5-b2d0-6d685132e6e8","error":"mTLS secrets not ready: certificate ray-head-cert-fail-k8s-job-g5dmq is not ready at current generation 1"}
{"level":"info","ts":"2026-11-03T15:22:21.472Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"1967bd07-ce60-44c5-b2d0-6d685132e6e8","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"fail-k8s-job-g5dmq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:22:21.472Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"1967bd07-ce60-44c5-b2d0-6d685132e6e8","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"fail-k8s-job-g5dmq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:22:21.472Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"1967bd07-ce60-44c5-b2d0-6d685132e6e8","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"fail-k8s-job-g5dmq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:22:21.472Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"1967bd07-ce60-44c5-b2d0-6d685132e6e8"}
{"level":"error","ts":"2026-11-03T15:22:21.472Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"1967bd07-ce60-44c5-b2d0-6d685132e6e8","error":"mTLS secrets not ready: certificate ray-head-cert-fail-k8s-job-g5dmq is not ready at current generation 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:22:21.615Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"72afc275-9b43-4498-b79d-c139630e61f3","certificate":"ray-head-cert-fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:22:21.615Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"72afc275-9b43-4498-b79d-c139630e61f3"}
{"level":"info","ts":"2026-11-03T15:22:21.615Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"1a2f0c90-c796-4a09-8bb9-d78f1bb022d3","cluster":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:22:21.793Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"3dc3cf34-d929-4300-a2c8-ee284d97d560"}
{"level":"info","ts":"2026-11-03T15:22:21.793Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"3dc3cf34-d929-4300-a2c8-ee284d97d560"}
{"level":"info","ts":"2026-11-03T15:22:21.793Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"3dc3cf34-d929-4300-a2c8-ee284d97d560","1 head service found":"fail-k8s-job-g5dmq-head-svc"}
{"level":"info","ts":"2026-11-03T15:22:21.793Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"3dc3cf34-d929-4300-a2c8-ee284d97d560"}
{"level":"info","ts":"2026-11-03T15:22:21.844Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"3dc3cf34-d929-4300-a2c8-ee284d97d560","error":"certificate ray-head-cert-fail-k8s-job-g5dmq is not ready at current generation 1"}
{"level":"error","ts":"2026-11-03T15:22:21.844Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"3dc3cf34-d929-4300-a2c8-ee284d97d560","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-fail-k8s-job-g5dmq is not ready at current generation 1","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:22:21.844Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"3dc3cf34-d929-4300-a2c8-ee284d97d560","error":"mTLS secrets not ready: certificate ray-head-cert-fail-k8s-job-g5dmq is not ready at current generation 1"}
{"level":"info","ts":"2026-11-03T15:22:21.844Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"3dc3cf34-d929-4300-a2c8-ee284d97d560","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"fail-k8s-job-g5dmq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:22:21.844Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"3dc3cf34-d929-4300-a2c8-ee284d97d560","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"fail-k8s-job-g5dmq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:22:21.844Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"3dc3cf34-d929-4300-a2c8-ee284d97d560","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"fail-k8s-job-g5dmq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:22:21.844Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"3dc3cf34-d929-4300-a2c8-ee284d97d560"}
{"level":"error","ts":"2026-11-03T15:22:21.844Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"3dc3cf34-d929-4300-a2c8-ee284d97d560","error":"mTLS secrets not ready: certificate ray-head-cert-fail-k8s-job-g5dmq is not ready at current generation 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:22:22.105Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"1a2f0c90-c796-4a09-8bb9-d78f1bb022d3","certificate":"ray-head-cert-fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:22:22.105Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"1a2f0c90-c796-4a09-8bb9-d78f1bb022d3"}
{"level":"info","ts":"2026-11-03T15:22:22.484Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"f03450f2-4d65-4d6e-9ce4-58aabe27a03d"}
{"level":"info","ts":"2026-11-03T15:22:22.484Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"f03450f2-4d65-4d6e-9ce4-58aabe27a03d"}
{"level":"info","ts":"2026-11-03T15:22:22.484Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"f03450f2-4d65-4d6e-9ce4-58aabe27a03d","1 head service found":"fail-k8s-job-g5dmq-head-svc"}
{"level":"info","ts":"2026-11-03T15:22:22.485Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"f03450f2-4d65-4d6e-9ce4-58aabe27a03d"}
{"level":"info","ts":"2026-11-03T15:22:22.536Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"f03450f2-4d65-4d6e-9ce4-58aabe27a03d","error":"certificate ray-head-cert-fail-k8s-job-g5dmq is not ready at current generation 1"}
{"level":"error","ts":"2026-11-03T15:22:22.536Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"f03450f2-4d65-4d6e-9ce4-58aabe27a03d","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-fail-k8s-job-g5dmq is not ready at current generation 1","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:22:22.536Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"f03450f2-4d65-4d6e-9ce4-58aabe27a03d","error":"mTLS secrets not ready: certificate ray-head-cert-fail-k8s-job-g5dmq is not ready at current generation 1"}
{"level":"info","ts":"2026-11-03T15:22:22.536Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"f03450f2-4d65-4d6e-9ce4-58aabe27a03d","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"fail-k8s-job-g5dmq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:22:22.536Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"f03450f2-4d65-4d6e-9ce4-58aabe27a03d","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"fail-k8s-job-g5dmq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:22:22.536Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"f03450f2-4d65-4d6e-9ce4-58aabe27a03d","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"fail-k8s-job-g5dmq","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:22:22.536Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"f03450f2-4d65-4d6e-9ce4-58aabe27a03d"}
{"level":"error","ts":"2026-11-03T15:22:22.536Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"f03450f2-4d65-4d6e-9ce4-58aabe27a03d","error":"mTLS secrets not ready: certificate ray-head-cert-fail-k8s-job-g5dmq is not ready at current generation 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:22:23.817Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"dd6a501c-0fda-4fbb-8736-812ff6a80ceb"}
{"level":"info","ts":"2026-11-03T15:22:23.817Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"dd6a501c-0fda-4fbb-8736-812ff6a80ceb"}
{"level":"info","ts":"2026-11-03T15:22:23.817Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"dd6a501c-0fda-4fbb-8736-812ff6a80ceb","1 head service found":"fail-k8s-job-g5dmq-head-svc"}
{"level":"info","ts":"2026-11-03T15:22:23.817Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"dd6a501c-0fda-4fbb-8736-812ff6a80ceb"}
{"level":"info","ts":"2026-11-03T15:22:23.922Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"dd6a501c-0fda-4fbb-8736-812ff6a80ceb","mode":"IntegratedOAuth","cluster":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:22:23.922Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"dd6a501c-0fda-4fbb-8736-812ff6a80ceb","cluster":"fail-k8s-job-g5dmq","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-11-03T15:22:23.922Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"dd6a501c-0fda-4fbb-8736-812ff6a80ceb","cluster":"fail-k8s-job-g5dmq","authType":"IntegratedOAuth","serviceAccount":"fail-k8s-job-g5dmq-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-11-03T15:22:23.922Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"dd6a501c-0fda-4fbb-8736-812ff6a80ceb","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"fail-k8s-job-g5dmq","ray.io/group":"headgroup","ray.io/identifier":"fail-k8s-job-g5dmq-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-11-03T15:22:23.922Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"dd6a501c-0fda-4fbb-8736-812ff6a80ceb","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-11-03T15:22:23.922Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"dd6a501c-0fda-4fbb-8736-812ff6a80ceb","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-11-03T15:22:23.922Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"dd6a501c-0fda-4fbb-8736-812ff6a80ceb","rayNodeType":"head","generatedCmd":"ulimit -n ${RAY_START_ULIMIT_OPEN_FILES:-65536}; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-11-03T15:22:23.922Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"dd6a501c-0fda-4fbb-8736-812ff6a80ceb","enabled":true}
{"level":"info","ts":"2026-11-03T15:22:24.033Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"dd6a501c-0fda-4fbb-8736-812ff6a80ceb","name":"fail-k8s-job-g5dmq-head-mhnw8"}
{"level":"info","ts":"2026-11-03T15:22:24.033Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"dd6a501c-0fda-4fbb-8736-812ff6a80ceb","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:22:24.033Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"dd6a501c-0fda-4fbb-8736-812ff6a80ceb","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:22:24.033Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"dd6a501c-0fda-4fbb-8736-812ff6a80ceb","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-11-03T15:22:24.033Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"dd6a501c-0fda-4fbb-8736-812ff6a80ceb","group":"small-group","indices":{}}
{"level":"info","ts":"2026-11-03T15:22:24.033Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"dd6a501c-0fda-4fbb-8736-812ff6a80ceb","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-11-03T15:22:24.135Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"dd6a501c-0fda-4fbb-8736-812ff6a80ceb","creating worker for group":"small-group","index":0,"total":1,"replicaIndex":0}
{"level":"info","ts":"2026-11-03T15:22:24.135Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"dd6a501c-0fda-4fbb-8736-812ff6a80ceb","nodeType":"worker","rayStartParams":{"address":"fail-k8s-job-g5dmq-head-svc.test-ns-kbx4w.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-11-03T15:22:24.135Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"dd6a501c-0fda-4fbb-8736-812ff6a80ceb","rayStartCmd":"ray start  --address=fail-k8s-job-g5dmq-head-svc.test-ns-kbx4w.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-11-03T15:22:24.135Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"dd6a501c-0fda-4fbb-8736-812ff6a80ceb","rayNodeType":"worker","generatedCmd":"ulimit -n ${RAY_START_ULIMIT_OPEN_FILES:-65536}; ray start  --address=fail-k8s-job-g5dmq-head-svc.test-ns-kbx4w.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-11-03T15:22:24.135Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"dd6a501c-0fda-4fbb-8736-812ff6a80ceb","enabled":true}
{"level":"info","ts":"2026-11-03T15:22:24.199Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"dd6a501c-0fda-4fbb-8736-812ff6a80ceb","name":"fail-k8s-job-g5dmq-small-group-worker-pf7h9"}
{"level":"info","ts":"2026-11-03T15:22:24.200Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"dd6a501c-0fda-4fbb-8736-812ff6a80ceb","podName":"fail-k8s-job-g5dmq-head-mhnw8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:22:24.200Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"dd6a501c-0fda-4fbb-8736-812ff6a80ceb","podName":"fail-k8s-job-g5dmq-head-mhnw8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:22:24.200Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"dd6a501c-0fda-4fbb-8736-812ff6a80ceb","name":"fail-k8s-job-g5dmq","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:24Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:22:24.263Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"f37f4a5b-a052-48d6-a440-f9dd2fbaf2d2"}
{"level":"info","ts":"2026-11-03T15:22:24.263Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"f37f4a5b-a052-48d6-a440-f9dd2fbaf2d2"}
{"level":"info","ts":"2026-11-03T15:22:24.263Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"f37f4a5b-a052-48d6-a440-f9dd2fbaf2d2","1 head service found":"fail-k8s-job-g5dmq-head-svc"}
{"level":"info","ts":"2026-11-03T15:22:24.264Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"eae9c7dd-cd84-494e-ae41-a0c2cc7a64a3","cluster":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:22:24.264Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"f37f4a5b-a052-48d6-a440-f9dd2fbaf2d2","Found 1 head Pod":"fail-k8s-job-g5dmq-head-mhnw8","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:22:24.264Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"f37f4a5b-a052-48d6-a440-f9dd2fbaf2d2","head Pod":"fail-k8s-job-g5dmq-head-mhnw8","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-g5dmq-head-mhnw8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:22:24.264Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"f37f4a5b-a052-48d6-a440-f9dd2fbaf2d2","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:22:24.264Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"f37f4a5b-a052-48d6-a440-f9dd2fbaf2d2","worker Pod":"fail-k8s-job-g5dmq-small-group-worker-pf7h9","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-g5dmq-small-group-worker-pf7h9. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:22:24.264Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"f37f4a5b-a052-48d6-a440-f9dd2fbaf2d2","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:22:24.264Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"f37f4a5b-a052-48d6-a440-f9dd2fbaf2d2","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:22:24.264Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"f37f4a5b-a052-48d6-a440-f9dd2fbaf2d2","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:22:24.264Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"f37f4a5b-a052-48d6-a440-f9dd2fbaf2d2","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:22:24.264Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"f37f4a5b-a052-48d6-a440-f9dd2fbaf2d2","podName":"fail-k8s-job-g5dmq-head-mhnw8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:22:24.264Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"481626ed-4ba0-41e5-bdf3-f9a9be2b20ba","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:22:24.264Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"481626ed-4ba0-41e5-bdf3-f9a9be2b20ba","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:22:24.264Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"f37f4a5b-a052-48d6-a440-f9dd2fbaf2d2","podName":"fail-k8s-job-g5dmq-head-mhnw8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:22:24.264Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"481626ed-4ba0-41e5-bdf3-f9a9be2b20ba","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:22:24.264Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"f37f4a5b-a052-48d6-a440-f9dd2fbaf2d2","name":"fail-k8s-job-g5dmq","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:24Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:22:24.264Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"481626ed-4ba0-41e5-bdf3-f9a9be2b20ba"}
{"level":"info","ts":"2026-11-03T15:22:24.264Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"481626ed-4ba0-41e5-bdf3-f9a9be2b20ba","RayCluster":"fail-k8s-job-g5dmq","State":""}
{"level":"info","ts":"2026-11-03T15:22:24.264Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"481626ed-4ba0-41e5-bdf3-f9a9be2b20ba","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:24Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:22:24.264Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"481626ed-4ba0-41e5-bdf3-f9a9be2b20ba","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:22:24.322Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"f37f4a5b-a052-48d6-a440-f9dd2fbaf2d2","name":"fail-k8s-job-g5dmq","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-k8s-job-g5dmq\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-kbx4w","name":"fail-k8s-job-g5dmq"}}
{"level":"info","ts":"2026-11-03T15:22:24.322Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"f37f4a5b-a052-48d6-a440-f9dd2fbaf2d2"}
{"level":"error","ts":"2026-11-03T15:22:24.322Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"f37f4a5b-a052-48d6-a440-f9dd2fbaf2d2","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-k8s-job-g5dmq\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:22:24.322Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"41765123-1e82-4955-b6a8-231441b54073"}
{"level":"info","ts":"2026-11-03T15:22:24.322Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"41765123-1e82-4955-b6a8-231441b54073"}
{"level":"info","ts":"2026-11-03T15:22:24.322Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"41765123-1e82-4955-b6a8-231441b54073","1 head service found":"fail-k8s-job-g5dmq-head-svc"}
{"level":"info","ts":"2026-11-03T15:22:24.322Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"41765123-1e82-4955-b6a8-231441b54073","Found 1 head Pod":"fail-k8s-job-g5dmq-head-mhnw8","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:22:24.322Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"41765123-1e82-4955-b6a8-231441b54073","head Pod":"fail-k8s-job-g5dmq-head-mhnw8","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-g5dmq-head-mhnw8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:22:24.322Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"41765123-1e82-4955-b6a8-231441b54073","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:22:24.322Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"41765123-1e82-4955-b6a8-231441b54073","worker Pod":"fail-k8s-job-g5dmq-small-group-worker-pf7h9","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-g5dmq-small-group-worker-pf7h9. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:22:24.322Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"41765123-1e82-4955-b6a8-231441b54073","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:22:24.322Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"41765123-1e82-4955-b6a8-231441b54073","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:22:24.322Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"41765123-1e82-4955-b6a8-231441b54073","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:22:24.322Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"41765123-1e82-4955-b6a8-231441b54073","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:22:24.322Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"41765123-1e82-4955-b6a8-231441b54073","podName":"fail-k8s-job-g5dmq-head-mhnw8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:22:24.322Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"41765123-1e82-4955-b6a8-231441b54073","podName":"fail-k8s-job-g5dmq-head-mhnw8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:22:24.322Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"41765123-1e82-4955-b6a8-231441b54073","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:22:24.322Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"41765123-1e82-4955-b6a8-231441b54073","seconds":300}
{"level":"info","ts":"2026-11-03T15:22:24.330Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"de0146a5-e1c3-4550-81ec-616921884413","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:22:24.330Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"de0146a5-e1c3-4550-81ec-616921884413","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:22:24.330Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"de0146a5-e1c3-4550-81ec-616921884413","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:22:24.330Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"de0146a5-e1c3-4550-81ec-616921884413"}
{"level":"info","ts":"2026-11-03T15:22:24.330Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"de0146a5-e1c3-4550-81ec-616921884413","RayCluster":"fail-k8s-job-g5dmq","State":""}
{"level":"info","ts":"2026-11-03T15:22:24.330Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"de0146a5-e1c3-4550-81ec-616921884413","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:24Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:24Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:22:24.363Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"85230211-357f-44f7-93fb-7631776e12c3"}
{"level":"info","ts":"2026-11-03T15:22:24.363Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"85230211-357f-44f7-93fb-7631776e12c3"}
{"level":"info","ts":"2026-11-03T15:22:24.363Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"85230211-357f-44f7-93fb-7631776e12c3","1 head service found":"fail-k8s-job-g5dmq-head-svc"}
{"level":"info","ts":"2026-11-03T15:22:24.363Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"85230211-357f-44f7-93fb-7631776e12c3","Found 1 head Pod":"fail-k8s-job-g5dmq-head-mhnw8","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:22:24.363Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"85230211-357f-44f7-93fb-7631776e12c3","head Pod":"fail-k8s-job-g5dmq-head-mhnw8","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-g5dmq-head-mhnw8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:22:24.363Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"85230211-357f-44f7-93fb-7631776e12c3","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:22:24.363Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"85230211-357f-44f7-93fb-7631776e12c3","worker Pod":"fail-k8s-job-g5dmq-small-group-worker-pf7h9","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-g5dmq-small-group-worker-pf7h9. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:22:24.363Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"85230211-357f-44f7-93fb-7631776e12c3","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:22:24.363Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"85230211-357f-44f7-93fb-7631776e12c3","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:22:24.363Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"85230211-357f-44f7-93fb-7631776e12c3","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:22:24.363Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"85230211-357f-44f7-93fb-7631776e12c3","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:22:24.364Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"85230211-357f-44f7-93fb-7631776e12c3","podName":"fail-k8s-job-g5dmq-head-mhnw8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:22:24.364Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"85230211-357f-44f7-93fb-7631776e12c3","podName":"fail-k8s-job-g5dmq-head-mhnw8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:22:24.364Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"85230211-357f-44f7-93fb-7631776e12c3","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:22:24.364Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"85230211-357f-44f7-93fb-7631776e12c3","seconds":300}
{"level":"info","ts":"2026-11-03T15:22:24.734Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"691e014b-0841-49e7-8236-bb141d039066"}
{"level":"info","ts":"2026-11-03T15:22:24.734Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"691e014b-0841-49e7-8236-bb141d039066"}
{"level":"info","ts":"2026-11-03T15:22:24.734Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"691e014b-0841-49e7-8236-bb141d039066","1 head service found":"fail-k8s-job-g5dmq-head-svc"}
{"level":"info","ts":"2026-11-03T15:22:24.734Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"691e014b-0841-49e7-8236-bb141d039066","Found 1 head Pod":"fail-k8s-job-g5dmq-head-mhnw8","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:22:24.734Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"691e014b-0841-49e7-8236-bb141d039066","head Pod":"fail-k8s-job-g5dmq-head-mhnw8","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-g5dmq-head-mhnw8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:22:24.734Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"691e014b-0841-49e7-8236-bb141d039066","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:22:24.734Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"691e014b-0841-49e7-8236-bb141d039066","worker Pod":"fail-k8s-job-g5dmq-small-group-worker-pf7h9","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-g5dmq-small-group-worker-pf7h9. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:22:24.734Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"691e014b-0841-49e7-8236-bb141d039066","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:22:24.734Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"691e014b-0841-49e7-8236-bb141d039066","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:22:24.734Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"691e014b-0841-49e7-8236-bb141d039066","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:22:24.734Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"691e014b-0841-49e7-8236-bb141d039066","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:22:24.734Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"691e014b-0841-49e7-8236-bb141d039066","podName":"fail-k8s-job-g5dmq-head-mhnw8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:22:24.734Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"691e014b-0841-49e7-8236-bb141d039066","podName":"fail-k8s-job-g5dmq-head-mhnw8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:22:24.734Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"691e014b-0841-49e7-8236-bb141d039066","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:22:24.734Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"691e014b-0841-49e7-8236-bb141d039066","seconds":300}
{"level":"info","ts":"2026-11-03T15:22:24.959Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"09db2462-b9ea-4488-ac7c-6e7ef35e9413"}
{"level":"info","ts":"2026-11-03T15:22:24.959Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"09db2462-b9ea-4488-ac7c-6e7ef35e9413"}
{"level":"info","ts":"2026-11-03T15:22:24.960Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"09db2462-b9ea-4488-ac7c-6e7ef35e9413","1 head service found":"fail-k8s-job-g5dmq-head-svc"}
{"level":"info","ts":"2026-11-03T15:22:24.960Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"09db2462-b9ea-4488-ac7c-6e7ef35e9413","Found 1 head Pod":"fail-k8s-job-g5dmq-head-mhnw8","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:22:24.960Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"09db2462-b9ea-4488-ac7c-6e7ef35e9413","head Pod":"fail-k8s-job-g5dmq-head-mhnw8","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-g5dmq-head-mhnw8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:22:24.960Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"09db2462-b9ea-4488-ac7c-6e7ef35e9413","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:22:24.960Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"09db2462-b9ea-4488-ac7c-6e7ef35e9413","worker Pod":"fail-k8s-job-g5dmq-small-group-worker-pf7h9","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-g5dmq-small-group-worker-pf7h9. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:22:24.960Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"09db2462-b9ea-4488-ac7c-6e7ef35e9413","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:22:24.960Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"09db2462-b9ea-4488-ac7c-6e7ef35e9413","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:22:24.960Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"09db2462-b9ea-4488-ac7c-6e7ef35e9413","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:22:24.960Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"09db2462-b9ea-4488-ac7c-6e7ef35e9413","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:22:24.960Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"09db2462-b9ea-4488-ac7c-6e7ef35e9413","podName":"fail-k8s-job-g5dmq-head-mhnw8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:22:24.960Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"09db2462-b9ea-4488-ac7c-6e7ef35e9413","podName":"fail-k8s-job-g5dmq-head-mhnw8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:22:24.960Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"09db2462-b9ea-4488-ac7c-6e7ef35e9413","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:22:24.960Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"09db2462-b9ea-4488-ac7c-6e7ef35e9413","seconds":300}
{"level":"info","ts":"2026-11-03T15:22:25.726Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"da453d5e-915f-4c86-aeec-9b07e8f41e3b"}
{"level":"info","ts":"2026-11-03T15:22:25.726Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"da453d5e-915f-4c86-aeec-9b07e8f41e3b"}
{"level":"info","ts":"2026-11-03T15:22:25.726Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"da453d5e-915f-4c86-aeec-9b07e8f41e3b","1 head service found":"fail-k8s-job-g5dmq-head-svc"}
{"level":"info","ts":"2026-11-03T15:22:25.726Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"da453d5e-915f-4c86-aeec-9b07e8f41e3b","Found 1 head Pod":"fail-k8s-job-g5dmq-head-mhnw8","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:22:25.726Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"da453d5e-915f-4c86-aeec-9b07e8f41e3b","head Pod":"fail-k8s-job-g5dmq-head-mhnw8","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-g5dmq-head-mhnw8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:22:25.726Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"da453d5e-915f-4c86-aeec-9b07e8f41e3b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:22:25.726Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"da453d5e-915f-4c86-aeec-9b07e8f41e3b","worker Pod":"fail-k8s-job-g5dmq-small-group-worker-pf7h9","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-g5dmq-small-group-worker-pf7h9. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:22:25.726Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"da453d5e-915f-4c86-aeec-9b07e8f41e3b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:22:25.726Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"da453d5e-915f-4c86-aeec-9b07e8f41e3b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:22:25.726Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"da453d5e-915f-4c86-aeec-9b07e8f41e3b","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:22:25.726Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"da453d5e-915f-4c86-aeec-9b07e8f41e3b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:22:25.727Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"da453d5e-915f-4c86-aeec-9b07e8f41e3b","podName":"fail-k8s-job-g5dmq-head-mhnw8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:22:25.727Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"da453d5e-915f-4c86-aeec-9b07e8f41e3b","podName":"fail-k8s-job-g5dmq-head-mhnw8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:22:25.727Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"da453d5e-915f-4c86-aeec-9b07e8f41e3b","name":"fail-k8s-job-g5dmq","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:24Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:22:25.788Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"a506c50c-9fc7-4a91-a93b-547ae8fdb555"}
{"level":"info","ts":"2026-11-03T15:22:25.788Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"a506c50c-9fc7-4a91-a93b-547ae8fdb555"}
{"level":"info","ts":"2026-11-03T15:22:25.788Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"a506c50c-9fc7-4a91-a93b-547ae8fdb555","1 head service found":"fail-k8s-job-g5dmq-head-svc"}
{"level":"info","ts":"2026-11-03T15:22:25.788Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"ede96b19-9b38-42a8-bf5f-cd76805d2337","cluster":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:22:25.788Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"a506c50c-9fc7-4a91-a93b-547ae8fdb555","Found 1 head Pod":"fail-k8s-job-g5dmq-head-mhnw8","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:22:25.788Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"a506c50c-9fc7-4a91-a93b-547ae8fdb555","head Pod":"fail-k8s-job-g5dmq-head-mhnw8","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-g5dmq-head-mhnw8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:22:25.788Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"a506c50c-9fc7-4a91-a93b-547ae8fdb555","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:22:25.788Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"a506c50c-9fc7-4a91-a93b-547ae8fdb555","worker Pod":"fail-k8s-job-g5dmq-small-group-worker-pf7h9","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-g5dmq-small-group-worker-pf7h9. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:22:25.788Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"a506c50c-9fc7-4a91-a93b-547ae8fdb555","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:22:25.788Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"a506c50c-9fc7-4a91-a93b-547ae8fdb555","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:22:25.788Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"a506c50c-9fc7-4a91-a93b-547ae8fdb555","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:22:25.788Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"a506c50c-9fc7-4a91-a93b-547ae8fdb555","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:22:25.788Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"a506c50c-9fc7-4a91-a93b-547ae8fdb555","podName":"fail-k8s-job-g5dmq-small-group-worker-pf7h9","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:22:25.788Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"a506c50c-9fc7-4a91-a93b-547ae8fdb555","podName":"fail-k8s-job-g5dmq-small-group-worker-pf7h9","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:22:25.788Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"c4693cb7-8cb1-4a40-9f52-990d57c272a8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:22:25.788Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"c4693cb7-8cb1-4a40-9f52-990d57c272a8","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:22:25.788Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"c4693cb7-8cb1-4a40-9f52-990d57c272a8","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:22:25.788Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"a506c50c-9fc7-4a91-a93b-547ae8fdb555","name":"fail-k8s-job-g5dmq","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:24Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:22:25.789Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"c4693cb7-8cb1-4a40-9f52-990d57c272a8"}
{"level":"info","ts":"2026-11-03T15:22:25.789Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"c4693cb7-8cb1-4a40-9f52-990d57c272a8","RayCluster":"fail-k8s-job-g5dmq","State":""}
{"level":"info","ts":"2026-11-03T15:22:25.789Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"c4693cb7-8cb1-4a40-9f52-990d57c272a8","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:24Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:22:25.789Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"c4693cb7-8cb1-4a40-9f52-990d57c272a8","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:22:25.844Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"a506c50c-9fc7-4a91-a93b-547ae8fdb555","name":"fail-k8s-job-g5dmq","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-k8s-job-g5dmq\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-kbx4w","name":"fail-k8s-job-g5dmq"}}
{"level":"info","ts":"2026-11-03T15:22:25.844Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"a506c50c-9fc7-4a91-a93b-547ae8fdb555"}
{"level":"error","ts":"2026-11-03T15:22:25.844Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"a506c50c-9fc7-4a91-a93b-547ae8fdb555","error":"Operation cannot be fulfilled on rayclusters.ray.io \"fail-k8s-job-g5dmq\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:22:25.850Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"178c121f-2632-4c87-a5df-96785985c772"}
{"level":"info","ts":"2026-11-03T15:22:25.850Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"178c121f-2632-4c87-a5df-96785985c772"}
{"level":"info","ts":"2026-11-03T15:22:25.850Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"178c121f-2632-4c87-a5df-96785985c772","1 head service found":"fail-k8s-job-g5dmq-head-svc"}
{"level":"info","ts":"2026-11-03T15:22:25.850Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"178c121f-2632-4c87-a5df-96785985c772","Found 1 head Pod":"fail-k8s-job-g5dmq-head-mhnw8","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:22:25.850Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"178c121f-2632-4c87-a5df-96785985c772","head Pod":"fail-k8s-job-g5dmq-head-mhnw8","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-g5dmq-head-mhnw8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:22:25.850Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"178c121f-2632-4c87-a5df-96785985c772","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:22:25.850Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"178c121f-2632-4c87-a5df-96785985c772","worker Pod":"fail-k8s-job-g5dmq-small-group-worker-pf7h9","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-g5dmq-small-group-worker-pf7h9. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:22:25.850Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"67c4d887-7796-4182-9fc1-c749b12951c7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:22:25.850Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"67c4d887-7796-4182-9fc1-c749b12951c7","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:22:25.850Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"67c4d887-7796-4182-9fc1-c749b12951c7","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:22:25.850Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"178c121f-2632-4c87-a5df-96785985c772","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:22:25.850Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"178c121f-2632-4c87-a5df-96785985c772","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:22:25.850Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"178c121f-2632-4c87-a5df-96785985c772","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:22:25.850Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"178c121f-2632-4c87-a5df-96785985c772","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:22:25.850Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"178c121f-2632-4c87-a5df-96785985c772","podName":"fail-k8s-job-g5dmq-small-group-worker-pf7h9","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:22:25.850Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"67c4d887-7796-4182-9fc1-c749b12951c7"}
{"level":"info","ts":"2026-11-03T15:22:25.850Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"67c4d887-7796-4182-9fc1-c749b12951c7","RayCluster":"fail-k8s-job-g5dmq","State":""}
{"level":"info","ts":"2026-11-03T15:22:25.850Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"178c121f-2632-4c87-a5df-96785985c772","podName":"fail-k8s-job-g5dmq-small-group-worker-pf7h9","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:22:25.850Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"67c4d887-7796-4182-9fc1-c749b12951c7","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:22:25.850Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"178c121f-2632-4c87-a5df-96785985c772","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:22:25.850Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"178c121f-2632-4c87-a5df-96785985c772","seconds":300}
{"level":"info","ts":"2026-11-03T15:22:26.095Z","logger":"controllers.RayClusterMTLS","msg":"Updating head certificate","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"ede96b19-9b38-42a8-bf5f-cd76805d2337","certificate":"ray-head-cert-fail-k8s-job-g5dmq","ipAddresses":["10.132.0.40","127.0.0.1"]}
{"level":"info","ts":"2026-11-03T15:22:26.360Z","logger":"controllers.RayClusterMTLS","msg":"Updating worker certificate","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"ede96b19-9b38-42a8-bf5f-cd76805d2337","certificate":"ray-worker-cert-fail-k8s-job-g5dmq","ipAddresses":["10.134.0.41","127.0.0.1"]}
{"level":"info","ts":"2026-11-03T15:22:26.621Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"ede96b19-9b38-42a8-bf5f-cd76805d2337","certificate":"ray-head-cert-fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:22:26.621Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"ede96b19-9b38-42a8-bf5f-cd76805d2337"}
{"level":"info","ts":"2026-11-03T15:22:28.851Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"37808385-83de-4b1b-9fb4-0e990beb2ebf","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:22:28.851Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"37808385-83de-4b1b-9fb4-0e990beb2ebf","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:22:28.851Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"37808385-83de-4b1b-9fb4-0e990beb2ebf","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:22:28.851Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"37808385-83de-4b1b-9fb4-0e990beb2ebf"}
{"level":"info","ts":"2026-11-03T15:22:28.851Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"37808385-83de-4b1b-9fb4-0e990beb2ebf","RayCluster":"fail-k8s-job-g5dmq","State":""}
{"level":"info","ts":"2026-11-03T15:22:28.851Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"37808385-83de-4b1b-9fb4-0e990beb2ebf","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:22:31.852Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"8c4041b0-89f1-4426-badb-1e6c5b8d9ab7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:22:31.852Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"8c4041b0-89f1-4426-badb-1e6c5b8d9ab7","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:22:31.852Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"8c4041b0-89f1-4426-badb-1e6c5b8d9ab7","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:22:31.852Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"8c4041b0-89f1-4426-badb-1e6c5b8d9ab7"}
{"level":"info","ts":"2026-11-03T15:22:31.852Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"8c4041b0-89f1-4426-badb-1e6c5b8d9ab7","RayCluster":"fail-k8s-job-g5dmq","State":""}
{"level":"info","ts":"2026-11-03T15:22:31.852Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"8c4041b0-89f1-4426-badb-1e6c5b8d9ab7","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:22:34.853Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"609b185a-6e05-44eb-8624-4c3c5be51e68","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:22:34.853Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"609b185a-6e05-44eb-8624-4c3c5be51e68","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:22:34.853Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"609b185a-6e05-44eb-8624-4c3c5be51e68","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:22:34.853Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"609b185a-6e05-44eb-8624-4c3c5be51e68"}
{"level":"info","ts":"2026-11-03T15:22:34.853Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"609b185a-6e05-44eb-8624-4c3c5be51e68","RayCluster":"fail-k8s-job-g5dmq","State":""}
{"level":"info","ts":"2026-11-03T15:22:34.853Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"609b185a-6e05-44eb-8624-4c3c5be51e68","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:22:37.854Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"33c2f3ba-9b95-423a-9c3a-5b00172a7064","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:22:37.854Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"33c2f3ba-9b95-423a-9c3a-5b00172a7064","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:22:37.854Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"33c2f3ba-9b95-423a-9c3a-5b00172a7064","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:22:37.854Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"33c2f3ba-9b95-423a-9c3a-5b00172a7064"}
{"level":"info","ts":"2026-11-03T15:22:37.854Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"33c2f3ba-9b95-423a-9c3a-5b00172a7064","RayCluster":"fail-k8s-job-g5dmq","State":""}
{"level":"info","ts":"2026-11-03T15:22:37.854Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"33c2f3ba-9b95-423a-9c3a-5b00172a7064","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:22:40.855Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"2b070273-c91b-4a46-8654-df040afc93f1","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:22:40.855Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"2b070273-c91b-4a46-8654-df040afc93f1","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:22:40.855Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"2b070273-c91b-4a46-8654-df040afc93f1","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:22:40.855Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"2b070273-c91b-4a46-8654-df040afc93f1"}
{"level":"info","ts":"2026-11-03T15:22:40.855Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"2b070273-c91b-4a46-8654-df040afc93f1","RayCluster":"fail-k8s-job-g5dmq","State":""}
{"level":"info","ts":"2026-11-03T15:22:40.855Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"2b070273-c91b-4a46-8654-df040afc93f1","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"error","ts":"2026-11-03T15:22:43.328Z","logger":"events","msg":"Server rejected event (will not retry!)","event":{"namespace":"test-ns-5lqjm","name":"counter-8j287.18e491d4966d5238"},"error":"namespaces \"test-ns-5lqjm\" not found","stacktrace":"k8s.io/client-go/tools/events.recordEvent\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:270\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).finishSeries\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:153\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext.func1\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:255\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:256\nk8s.io/apimachinery/pkg/util/wait.JitterUntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:223\nk8s.io/apimachinery/pkg/util/wait.UntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:172"}
{"level":"error","ts":"2026-11-03T15:22:43.480Z","logger":"events","msg":"Server rejected event (will not retry!)","event":{"namespace":"test-ns-5lqjm","name":"counter-8j287.18e491d5d83e5dc2"},"error":"namespaces \"test-ns-5lqjm\" not found","stacktrace":"k8s.io/client-go/tools/events.recordEvent\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:270\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).finishSeries\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:153\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext.func1\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:255\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:256\nk8s.io/apimachinery/pkg/util/wait.JitterUntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:223\nk8s.io/apimachinery/pkg/util/wait.UntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:172"}
{"level":"error","ts":"2026-11-03T15:22:43.631Z","logger":"events","msg":"Server rejected event (will not retry!)","event":{"namespace":"test-ns-5lqjm","name":"long-running-7qq77.18e491bd91baf2dc"},"error":"namespaces \"test-ns-5lqjm\" not found","stacktrace":"k8s.io/client-go/tools/events.recordEvent\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:270\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).finishSeries\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:153\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext.func1\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:255\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:256\nk8s.io/apimachinery/pkg/util/wait.JitterUntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:223\nk8s.io/apimachinery/pkg/util/wait.UntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:172"}
{"level":"error","ts":"2026-11-03T15:22:43.783Z","logger":"events","msg":"Server rejected event (will not retry!)","event":{"namespace":"test-ns-5lqjm","name":"long-running-2zj8b.18e491a11c6c7e6e"},"error":"namespaces \"test-ns-5lqjm\" not found","stacktrace":"k8s.io/client-go/tools/events.recordEvent\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:270\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).finishSeries\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:153\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext.func1\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:255\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:256\nk8s.io/apimachinery/pkg/util/wait.JitterUntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:223\nk8s.io/apimachinery/pkg/util/wait.UntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:172"}
{"level":"info","ts":"2026-11-03T15:22:43.856Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"f9bcda02-1538-4987-9898-9e7fb73076d6","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:22:43.857Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"f9bcda02-1538-4987-9898-9e7fb73076d6","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:22:43.857Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"f9bcda02-1538-4987-9898-9e7fb73076d6","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:22:43.857Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"f9bcda02-1538-4987-9898-9e7fb73076d6"}
{"level":"info","ts":"2026-11-03T15:22:43.857Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"f9bcda02-1538-4987-9898-9e7fb73076d6","RayCluster":"fail-k8s-job-g5dmq","State":""}
{"level":"info","ts":"2026-11-03T15:22:43.857Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"f9bcda02-1538-4987-9898-9e7fb73076d6","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"error","ts":"2026-11-03T15:22:43.937Z","logger":"events","msg":"Server rejected event (will not retry!)","event":{"namespace":"test-ns-5lqjm","name":"long-running-2zj8b.18e491a2f59e9e58"},"error":"namespaces \"test-ns-5lqjm\" not found","stacktrace":"k8s.io/client-go/tools/events.recordEvent\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:270\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).finishSeries\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:153\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext.func1\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:255\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:256\nk8s.io/apimachinery/pkg/util/wait.JitterUntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:223\nk8s.io/apimachinery/pkg/util/wait.UntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:172"}
{"level":"error","ts":"2026-11-03T15:22:44.089Z","logger":"events","msg":"Server rejected event (will not retry!)","event":{"namespace":"test-ns-5lqjm","name":"long-running-7qq77.18e491beb082b683"},"error":"namespaces \"test-ns-5lqjm\" not found","stacktrace":"k8s.io/client-go/tools/events.recordEvent\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:270\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).finishSeries\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:153\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext.func1\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:255\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:256\nk8s.io/apimachinery/pkg/util/wait.JitterUntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:223\nk8s.io/apimachinery/pkg/util/wait.UntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:172"}
{"level":"error","ts":"2026-11-03T15:22:44.241Z","logger":"events","msg":"Server rejected event (will not retry!)","event":{"namespace":"test-ns-5lqjm","name":"long-running-7qq77.18e491bec90a99ba"},"error":"namespaces \"test-ns-5lqjm\" not found","stacktrace":"k8s.io/client-go/tools/events.recordEvent\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:270\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).finishSeries\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:153\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext.func1\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:255\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:256\nk8s.io/apimachinery/pkg/util/wait.JitterUntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:223\nk8s.io/apimachinery/pkg/util/wait.UntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:172"}
{"level":"error","ts":"2026-11-03T15:22:44.395Z","logger":"events","msg":"Server rejected event (will not retry!)","event":{"namespace":"test-ns-5lqjm","name":"long-running-2zj8b.18e491a2dd4311f9"},"error":"namespaces \"test-ns-5lqjm\" not found","stacktrace":"k8s.io/client-go/tools/events.recordEvent\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:270\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).finishSeries\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:153\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext.func1\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:255\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:256\nk8s.io/apimachinery/pkg/util/wait.JitterUntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:223\nk8s.io/apimachinery/pkg/util/wait.UntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:172"}
{"level":"info","ts":"2026-11-03T15:22:46.858Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"86552d50-7c1e-4e09-b9c7-ab9ed3fb1bc6","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:22:46.858Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"86552d50-7c1e-4e09-b9c7-ab9ed3fb1bc6","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:22:46.858Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"86552d50-7c1e-4e09-b9c7-ab9ed3fb1bc6","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:22:46.858Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"86552d50-7c1e-4e09-b9c7-ab9ed3fb1bc6"}
{"level":"info","ts":"2026-11-03T15:22:46.858Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"86552d50-7c1e-4e09-b9c7-ab9ed3fb1bc6","RayCluster":"fail-k8s-job-g5dmq","State":""}
{"level":"info","ts":"2026-11-03T15:22:46.858Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"86552d50-7c1e-4e09-b9c7-ab9ed3fb1bc6","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:22:49.859Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"3b7e1996-d16f-4e32-92ed-89f691880a63","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:22:49.859Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"3b7e1996-d16f-4e32-92ed-89f691880a63","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:22:49.859Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"3b7e1996-d16f-4e32-92ed-89f691880a63","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:22:49.860Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"3b7e1996-d16f-4e32-92ed-89f691880a63"}
{"level":"info","ts":"2026-11-03T15:22:49.860Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"3b7e1996-d16f-4e32-92ed-89f691880a63","RayCluster":"fail-k8s-job-g5dmq","State":""}
{"level":"info","ts":"2026-11-03T15:22:49.860Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"3b7e1996-d16f-4e32-92ed-89f691880a63","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:22:52.860Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"38ff6d03-b0a9-4bff-bc59-cb78411ff2cf","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:22:52.860Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"38ff6d03-b0a9-4bff-bc59-cb78411ff2cf","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:22:52.860Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"38ff6d03-b0a9-4bff-bc59-cb78411ff2cf","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:22:52.860Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"38ff6d03-b0a9-4bff-bc59-cb78411ff2cf"}
{"level":"info","ts":"2026-11-03T15:22:52.860Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"38ff6d03-b0a9-4bff-bc59-cb78411ff2cf","RayCluster":"fail-k8s-job-g5dmq","State":""}
{"level":"info","ts":"2026-11-03T15:22:52.860Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"38ff6d03-b0a9-4bff-bc59-cb78411ff2cf","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:22:55.861Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"e4473a3e-8b71-4ab9-9423-07c1ad1e4894","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:22:55.861Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"e4473a3e-8b71-4ab9-9423-07c1ad1e4894","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:22:55.861Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"e4473a3e-8b71-4ab9-9423-07c1ad1e4894","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:22:55.861Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"e4473a3e-8b71-4ab9-9423-07c1ad1e4894"}
{"level":"info","ts":"2026-11-03T15:22:55.861Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"e4473a3e-8b71-4ab9-9423-07c1ad1e4894","RayCluster":"fail-k8s-job-g5dmq","State":""}
{"level":"info","ts":"2026-11-03T15:22:55.861Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"e4473a3e-8b71-4ab9-9423-07c1ad1e4894","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:22:56.622Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"6f0d1b7a-7dd9-40a2-93c9-213b9e055e57","cluster":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:22:58.862Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"a97a895e-83f5-4518-a6d3-ad25d2653f04","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:22:58.862Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"a97a895e-83f5-4518-a6d3-ad25d2653f04","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:22:58.862Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"a97a895e-83f5-4518-a6d3-ad25d2653f04","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:22:58.863Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"a97a895e-83f5-4518-a6d3-ad25d2653f04"}
{"level":"info","ts":"2026-11-03T15:22:58.863Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"a97a895e-83f5-4518-a6d3-ad25d2653f04","RayCluster":"fail-k8s-job-g5dmq","State":""}
{"level":"info","ts":"2026-11-03T15:22:58.863Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"a97a895e-83f5-4518-a6d3-ad25d2653f04","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:23:01.864Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"079e0de5-1d04-4ba3-a478-7ae70210b3d7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:23:01.864Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"079e0de5-1d04-4ba3-a478-7ae70210b3d7","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:23:01.864Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"079e0de5-1d04-4ba3-a478-7ae70210b3d7","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:23:01.864Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"079e0de5-1d04-4ba3-a478-7ae70210b3d7"}
{"level":"info","ts":"2026-11-03T15:23:01.864Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"079e0de5-1d04-4ba3-a478-7ae70210b3d7","RayCluster":"fail-k8s-job-g5dmq","State":""}
{"level":"info","ts":"2026-11-03T15:23:01.864Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"079e0de5-1d04-4ba3-a478-7ae70210b3d7","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:23:04.866Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"8fb27826-8ecc-4955-9516-a3d4d2b815e8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:23:04.866Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"8fb27826-8ecc-4955-9516-a3d4d2b815e8","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:23:04.866Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"8fb27826-8ecc-4955-9516-a3d4d2b815e8","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:23:04.866Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"8fb27826-8ecc-4955-9516-a3d4d2b815e8"}
{"level":"info","ts":"2026-11-03T15:23:04.866Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"8fb27826-8ecc-4955-9516-a3d4d2b815e8","RayCluster":"fail-k8s-job-g5dmq","State":""}
{"level":"info","ts":"2026-11-03T15:23:04.866Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"8fb27826-8ecc-4955-9516-a3d4d2b815e8","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:23:07.867Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"8fa7a12a-3e08-4d4b-8a83-5003a3c62f66","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:23:07.867Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"8fa7a12a-3e08-4d4b-8a83-5003a3c62f66","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:23:07.867Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"8fa7a12a-3e08-4d4b-8a83-5003a3c62f66","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:23:07.867Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"8fa7a12a-3e08-4d4b-8a83-5003a3c62f66"}
{"level":"info","ts":"2026-11-03T15:23:07.867Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"8fa7a12a-3e08-4d4b-8a83-5003a3c62f66","RayCluster":"fail-k8s-job-g5dmq","State":""}
{"level":"info","ts":"2026-11-03T15:23:07.867Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"8fa7a12a-3e08-4d4b-8a83-5003a3c62f66","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:23:10.868Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"8d54e2ef-97d2-433c-960e-c89e4553cad9","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:23:10.868Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"8d54e2ef-97d2-433c-960e-c89e4553cad9","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:23:10.868Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"8d54e2ef-97d2-433c-960e-c89e4553cad9","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:23:10.868Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"8d54e2ef-97d2-433c-960e-c89e4553cad9"}
{"level":"info","ts":"2026-11-03T15:23:10.868Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"8d54e2ef-97d2-433c-960e-c89e4553cad9","RayCluster":"fail-k8s-job-g5dmq","State":""}
{"level":"info","ts":"2026-11-03T15:23:10.868Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"8d54e2ef-97d2-433c-960e-c89e4553cad9","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:23:13.869Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"82ea5437-3ad4-4e22-9bb5-447e49a34181","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:23:13.869Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"82ea5437-3ad4-4e22-9bb5-447e49a34181","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:23:13.869Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"82ea5437-3ad4-4e22-9bb5-447e49a34181","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:23:13.869Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"82ea5437-3ad4-4e22-9bb5-447e49a34181"}
{"level":"info","ts":"2026-11-03T15:23:13.869Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"82ea5437-3ad4-4e22-9bb5-447e49a34181","RayCluster":"fail-k8s-job-g5dmq","State":""}
{"level":"info","ts":"2026-11-03T15:23:13.869Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"82ea5437-3ad4-4e22-9bb5-447e49a34181","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:23:16.871Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"63dc8b04-baf9-4cca-87ec-0db7556aab0f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:23:16.871Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"63dc8b04-baf9-4cca-87ec-0db7556aab0f","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:23:16.871Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"63dc8b04-baf9-4cca-87ec-0db7556aab0f","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:23:16.871Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"63dc8b04-baf9-4cca-87ec-0db7556aab0f"}
{"level":"info","ts":"2026-11-03T15:23:16.871Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"63dc8b04-baf9-4cca-87ec-0db7556aab0f","RayCluster":"fail-k8s-job-g5dmq","State":""}
{"level":"info","ts":"2026-11-03T15:23:16.871Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"63dc8b04-baf9-4cca-87ec-0db7556aab0f","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:23:19.872Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"93a15e94-354c-454c-af76-c9bcea425434","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:23:19.872Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"93a15e94-354c-454c-af76-c9bcea425434","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:23:19.872Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"93a15e94-354c-454c-af76-c9bcea425434","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:23:19.872Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"93a15e94-354c-454c-af76-c9bcea425434"}
{"level":"info","ts":"2026-11-03T15:23:19.872Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"93a15e94-354c-454c-af76-c9bcea425434","RayCluster":"fail-k8s-job-g5dmq","State":""}
{"level":"info","ts":"2026-11-03T15:23:19.872Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"93a15e94-354c-454c-af76-c9bcea425434","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:23:22.873Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"71f06218-b966-4b6d-859a-931bac4a4263","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:23:22.873Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"71f06218-b966-4b6d-859a-931bac4a4263","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:23:22.873Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"71f06218-b966-4b6d-859a-931bac4a4263","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:23:22.873Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"71f06218-b966-4b6d-859a-931bac4a4263"}
{"level":"info","ts":"2026-11-03T15:23:22.873Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"71f06218-b966-4b6d-859a-931bac4a4263","RayCluster":"fail-k8s-job-g5dmq","State":""}
{"level":"info","ts":"2026-11-03T15:23:22.873Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"71f06218-b966-4b6d-859a-931bac4a4263","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:23:25.874Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"d5f2d5f5-7891-4e49-b5e2-f264325c4ac2","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:23:25.874Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"d5f2d5f5-7891-4e49-b5e2-f264325c4ac2","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:23:25.875Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"d5f2d5f5-7891-4e49-b5e2-f264325c4ac2","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:23:25.875Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"d5f2d5f5-7891-4e49-b5e2-f264325c4ac2"}
{"level":"info","ts":"2026-11-03T15:23:25.875Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"d5f2d5f5-7891-4e49-b5e2-f264325c4ac2","RayCluster":"fail-k8s-job-g5dmq","State":""}
{"level":"info","ts":"2026-11-03T15:23:25.875Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"d5f2d5f5-7891-4e49-b5e2-f264325c4ac2","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:23:28.876Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"71aa5834-9f23-4f89-83ed-89c01a15a6bc","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:23:28.876Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"71aa5834-9f23-4f89-83ed-89c01a15a6bc","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:23:28.876Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"71aa5834-9f23-4f89-83ed-89c01a15a6bc","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:23:28.876Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"71aa5834-9f23-4f89-83ed-89c01a15a6bc"}
{"level":"info","ts":"2026-11-03T15:23:28.876Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"71aa5834-9f23-4f89-83ed-89c01a15a6bc","RayCluster":"fail-k8s-job-g5dmq","State":""}
{"level":"info","ts":"2026-11-03T15:23:28.876Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"71aa5834-9f23-4f89-83ed-89c01a15a6bc","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:23:31.877Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"4c805890-323c-4952-b86e-1e435df7ff60","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:23:31.877Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"4c805890-323c-4952-b86e-1e435df7ff60","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:23:31.877Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"4c805890-323c-4952-b86e-1e435df7ff60","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:23:31.877Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"4c805890-323c-4952-b86e-1e435df7ff60"}
{"level":"info","ts":"2026-11-03T15:23:31.877Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"4c805890-323c-4952-b86e-1e435df7ff60","RayCluster":"fail-k8s-job-g5dmq","State":""}
{"level":"info","ts":"2026-11-03T15:23:31.877Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"4c805890-323c-4952-b86e-1e435df7ff60","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:23:34.878Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"7810d756-5a17-484b-8d70-51056f231e58","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:23:34.878Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"7810d756-5a17-484b-8d70-51056f231e58","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:23:34.878Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"7810d756-5a17-484b-8d70-51056f231e58","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:23:34.878Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"7810d756-5a17-484b-8d70-51056f231e58"}
{"level":"info","ts":"2026-11-03T15:23:34.878Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"7810d756-5a17-484b-8d70-51056f231e58","RayCluster":"fail-k8s-job-g5dmq","State":""}
{"level":"info","ts":"2026-11-03T15:23:34.878Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"7810d756-5a17-484b-8d70-51056f231e58","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:23:37.879Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"77dbf0cc-e962-4aef-a089-78039b184c4b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:23:37.879Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"77dbf0cc-e962-4aef-a089-78039b184c4b","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:23:37.879Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"77dbf0cc-e962-4aef-a089-78039b184c4b","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:23:37.879Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"77dbf0cc-e962-4aef-a089-78039b184c4b"}
{"level":"info","ts":"2026-11-03T15:23:37.879Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"77dbf0cc-e962-4aef-a089-78039b184c4b","RayCluster":"fail-k8s-job-g5dmq","State":""}
{"level":"info","ts":"2026-11-03T15:23:37.879Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"77dbf0cc-e962-4aef-a089-78039b184c4b","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:23:40.881Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"50961e5e-9b6a-47b5-b0a2-03bf071ea081","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:23:40.881Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"50961e5e-9b6a-47b5-b0a2-03bf071ea081","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:23:40.881Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"50961e5e-9b6a-47b5-b0a2-03bf071ea081","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:23:40.881Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"50961e5e-9b6a-47b5-b0a2-03bf071ea081"}
{"level":"info","ts":"2026-11-03T15:23:40.881Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"50961e5e-9b6a-47b5-b0a2-03bf071ea081","RayCluster":"fail-k8s-job-g5dmq","State":""}
{"level":"info","ts":"2026-11-03T15:23:40.881Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"50961e5e-9b6a-47b5-b0a2-03bf071ea081","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:23:41.072Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"c55babf5-5772-45f4-b580-3432749b347f"}
{"level":"info","ts":"2026-11-03T15:23:41.072Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"c55babf5-5772-45f4-b580-3432749b347f"}
{"level":"info","ts":"2026-11-03T15:23:41.072Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"c55babf5-5772-45f4-b580-3432749b347f","1 head service found":"fail-k8s-job-g5dmq-head-svc"}
{"level":"info","ts":"2026-11-03T15:23:41.072Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"c55babf5-5772-45f4-b580-3432749b347f","Found 1 head Pod":"fail-k8s-job-g5dmq-head-mhnw8","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:23:41.072Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"c55babf5-5772-45f4-b580-3432749b347f","head Pod":"fail-k8s-job-g5dmq-head-mhnw8","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-g5dmq-head-mhnw8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:23:41.072Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"c55babf5-5772-45f4-b580-3432749b347f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:23:41.072Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"c55babf5-5772-45f4-b580-3432749b347f","worker Pod":"fail-k8s-job-g5dmq-small-group-worker-pf7h9","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-g5dmq-small-group-worker-pf7h9. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:23:41.072Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"c55babf5-5772-45f4-b580-3432749b347f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:23:41.072Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"c55babf5-5772-45f4-b580-3432749b347f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:23:41.072Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"c55babf5-5772-45f4-b580-3432749b347f","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:23:41.072Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"c55babf5-5772-45f4-b580-3432749b347f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:23:41.072Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"c55babf5-5772-45f4-b580-3432749b347f","podName":"fail-k8s-job-g5dmq-small-group-worker-pf7h9","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:23:41.072Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"c55babf5-5772-45f4-b580-3432749b347f","podName":"fail-k8s-job-g5dmq-small-group-worker-pf7h9","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:23:41.073Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"c55babf5-5772-45f4-b580-3432749b347f","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:23:41.073Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"c55babf5-5772-45f4-b580-3432749b347f","seconds":300}
{"level":"info","ts":"2026-11-03T15:23:42.073Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"a14f923e-ece9-4cfb-8fb0-8bd6eac65356"}
{"level":"info","ts":"2026-11-03T15:23:42.073Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"a14f923e-ece9-4cfb-8fb0-8bd6eac65356"}
{"level":"info","ts":"2026-11-03T15:23:42.073Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"a14f923e-ece9-4cfb-8fb0-8bd6eac65356","1 head service found":"fail-k8s-job-g5dmq-head-svc"}
{"level":"info","ts":"2026-11-03T15:23:42.073Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"a14f923e-ece9-4cfb-8fb0-8bd6eac65356","Found 1 head Pod":"fail-k8s-job-g5dmq-head-mhnw8","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:23:42.073Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"a14f923e-ece9-4cfb-8fb0-8bd6eac65356","head Pod":"fail-k8s-job-g5dmq-head-mhnw8","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-g5dmq-head-mhnw8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:23:42.073Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"a14f923e-ece9-4cfb-8fb0-8bd6eac65356","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:23:42.073Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"a14f923e-ece9-4cfb-8fb0-8bd6eac65356","worker Pod":"fail-k8s-job-g5dmq-small-group-worker-pf7h9","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-g5dmq-small-group-worker-pf7h9. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:23:42.073Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"a14f923e-ece9-4cfb-8fb0-8bd6eac65356","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:23:42.073Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"a14f923e-ece9-4cfb-8fb0-8bd6eac65356","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:23:42.073Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"a14f923e-ece9-4cfb-8fb0-8bd6eac65356","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:23:42.074Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"a14f923e-ece9-4cfb-8fb0-8bd6eac65356","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:23:42.074Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"a14f923e-ece9-4cfb-8fb0-8bd6eac65356","podName":"fail-k8s-job-g5dmq-head-mhnw8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:23:42.074Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"a14f923e-ece9-4cfb-8fb0-8bd6eac65356","podName":"fail-k8s-job-g5dmq-head-mhnw8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:23:42.074Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"a14f923e-ece9-4cfb-8fb0-8bd6eac65356","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:23:42.074Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"a14f923e-ece9-4cfb-8fb0-8bd6eac65356","seconds":300}
{"level":"info","ts":"2026-11-03T15:23:43.882Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"9562ac4d-6b74-4625-8a30-3b7a302f5eed","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:23:43.882Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"9562ac4d-6b74-4625-8a30-3b7a302f5eed","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:23:43.882Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"9562ac4d-6b74-4625-8a30-3b7a302f5eed","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:23:43.882Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"9562ac4d-6b74-4625-8a30-3b7a302f5eed"}
{"level":"info","ts":"2026-11-03T15:23:43.882Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"9562ac4d-6b74-4625-8a30-3b7a302f5eed","RayCluster":"fail-k8s-job-g5dmq","State":""}
{"level":"info","ts":"2026-11-03T15:23:43.882Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"9562ac4d-6b74-4625-8a30-3b7a302f5eed","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:23:46.883Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"69dd8471-831c-4699-91ec-8403395306ad","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:23:46.883Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"69dd8471-831c-4699-91ec-8403395306ad","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:23:46.883Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"69dd8471-831c-4699-91ec-8403395306ad","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:23:46.883Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"69dd8471-831c-4699-91ec-8403395306ad"}
{"level":"info","ts":"2026-11-03T15:23:46.883Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"69dd8471-831c-4699-91ec-8403395306ad","RayCluster":"fail-k8s-job-g5dmq","State":""}
{"level":"info","ts":"2026-11-03T15:23:46.883Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"69dd8471-831c-4699-91ec-8403395306ad","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:23:49.884Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"76c41211-59bb-4c11-862c-07009490071c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:23:49.884Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"76c41211-59bb-4c11-862c-07009490071c","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:23:49.884Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"76c41211-59bb-4c11-862c-07009490071c","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:23:49.884Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"76c41211-59bb-4c11-862c-07009490071c"}
{"level":"info","ts":"2026-11-03T15:23:49.884Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"76c41211-59bb-4c11-862c-07009490071c","RayCluster":"fail-k8s-job-g5dmq","State":""}
{"level":"info","ts":"2026-11-03T15:23:49.884Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"76c41211-59bb-4c11-862c-07009490071c","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:23:51.111Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"e286daba-0cbc-453a-a824-f2ddb121df6d"}
{"level":"info","ts":"2026-11-03T15:23:51.111Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"e286daba-0cbc-453a-a824-f2ddb121df6d"}
{"level":"info","ts":"2026-11-03T15:23:51.111Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"e286daba-0cbc-453a-a824-f2ddb121df6d","1 head service found":"fail-k8s-job-g5dmq-head-svc"}
{"level":"info","ts":"2026-11-03T15:23:51.111Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"e286daba-0cbc-453a-a824-f2ddb121df6d","Found 1 head Pod":"fail-k8s-job-g5dmq-head-mhnw8","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:23:51.111Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"e286daba-0cbc-453a-a824-f2ddb121df6d","head Pod":"fail-k8s-job-g5dmq-head-mhnw8","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-g5dmq-head-mhnw8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:23:51.111Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"e286daba-0cbc-453a-a824-f2ddb121df6d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:23:51.111Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"e286daba-0cbc-453a-a824-f2ddb121df6d","worker Pod":"fail-k8s-job-g5dmq-small-group-worker-pf7h9","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-g5dmq-small-group-worker-pf7h9. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:23:51.111Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"e286daba-0cbc-453a-a824-f2ddb121df6d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:23:51.111Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"e286daba-0cbc-453a-a824-f2ddb121df6d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:23:51.111Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"e286daba-0cbc-453a-a824-f2ddb121df6d","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:23:51.111Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"e286daba-0cbc-453a-a824-f2ddb121df6d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:23:51.111Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"e286daba-0cbc-453a-a824-f2ddb121df6d","podName":"fail-k8s-job-g5dmq-head-mhnw8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:23:51.111Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"e286daba-0cbc-453a-a824-f2ddb121df6d","podName":"fail-k8s-job-g5dmq-head-mhnw8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:23:51.111Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"e286daba-0cbc-453a-a824-f2ddb121df6d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:23:51.111Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"e286daba-0cbc-453a-a824-f2ddb121df6d","seconds":300}
{"level":"info","ts":"2026-11-03T15:23:52.128Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"d0dd65d3-a0d6-4353-8ca5-6c7bc759163d"}
{"level":"info","ts":"2026-11-03T15:23:52.128Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"d0dd65d3-a0d6-4353-8ca5-6c7bc759163d"}
{"level":"info","ts":"2026-11-03T15:23:52.128Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"d0dd65d3-a0d6-4353-8ca5-6c7bc759163d","1 head service found":"fail-k8s-job-g5dmq-head-svc"}
{"level":"info","ts":"2026-11-03T15:23:52.129Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"d0dd65d3-a0d6-4353-8ca5-6c7bc759163d","Found 1 head Pod":"fail-k8s-job-g5dmq-head-mhnw8","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:23:52.129Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"d0dd65d3-a0d6-4353-8ca5-6c7bc759163d","head Pod":"fail-k8s-job-g5dmq-head-mhnw8","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-g5dmq-head-mhnw8. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:23:52.129Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"d0dd65d3-a0d6-4353-8ca5-6c7bc759163d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:23:52.129Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"d0dd65d3-a0d6-4353-8ca5-6c7bc759163d","worker Pod":"fail-k8s-job-g5dmq-small-group-worker-pf7h9","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-g5dmq-small-group-worker-pf7h9. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:23:52.129Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"d0dd65d3-a0d6-4353-8ca5-6c7bc759163d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:23:52.129Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"d0dd65d3-a0d6-4353-8ca5-6c7bc759163d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:23:52.129Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"d0dd65d3-a0d6-4353-8ca5-6c7bc759163d","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:23:52.129Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"d0dd65d3-a0d6-4353-8ca5-6c7bc759163d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:23:52.129Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"d0dd65d3-a0d6-4353-8ca5-6c7bc759163d","podName":"fail-k8s-job-g5dmq-small-group-worker-pf7h9","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:23:52.129Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"d0dd65d3-a0d6-4353-8ca5-6c7bc759163d","podName":"fail-k8s-job-g5dmq-small-group-worker-pf7h9","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:23:52.129Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"d0dd65d3-a0d6-4353-8ca5-6c7bc759163d","name":"fail-k8s-job-g5dmq","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:23:52Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:23:52.197Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"5950cd05-5211-4449-9260-f5d180f0596e","cluster":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:23:52.197Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"d99d1175-1a93-4a39-a2ff-a285fc85a97e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:23:52.197Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"d99d1175-1a93-4a39-a2ff-a285fc85a97e","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:23:52.197Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"d99d1175-1a93-4a39-a2ff-a285fc85a97e","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:23:52.198Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"d99d1175-1a93-4a39-a2ff-a285fc85a97e"}
{"level":"info","ts":"2026-11-03T15:23:52.198Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"d99d1175-1a93-4a39-a2ff-a285fc85a97e","RayCluster":"fail-k8s-job-g5dmq","State":""}
{"level":"info","ts":"2026-11-03T15:23:52.198Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"d99d1175-1a93-4a39-a2ff-a285fc85a97e","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:22:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:23:52Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:23:52.198Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"d99d1175-1a93-4a39-a2ff-a285fc85a97e","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:23:52.260Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"9bc0bff3-1d31-4228-ad78-cce3f7cbf96e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:23:52.260Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"9bc0bff3-1d31-4228-ad78-cce3f7cbf96e","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:23:52.260Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"9bc0bff3-1d31-4228-ad78-cce3f7cbf96e","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:23:52.260Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"9bc0bff3-1d31-4228-ad78-cce3f7cbf96e"}
{"level":"info","ts":"2026-11-03T15:23:52.260Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"9bc0bff3-1d31-4228-ad78-cce3f7cbf96e","RayCluster":"fail-k8s-job-g5dmq","State":""}
{"level":"info","ts":"2026-11-03T15:23:52.260Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"9bc0bff3-1d31-4228-ad78-cce3f7cbf96e","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:23:52Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:23:52Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:23:54.198Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"ab6229c6-4e26-4495-bd24-f332c7160395"}
{"level":"info","ts":"2026-11-03T15:23:54.198Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"ab6229c6-4e26-4495-bd24-f332c7160395"}
{"level":"info","ts":"2026-11-03T15:23:54.198Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"ab6229c6-4e26-4495-bd24-f332c7160395","1 head service found":"fail-k8s-job-g5dmq-head-svc"}
{"level":"info","ts":"2026-11-03T15:23:54.198Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"ab6229c6-4e26-4495-bd24-f332c7160395","Found 1 head Pod":"fail-k8s-job-g5dmq-head-mhnw8","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:23:54.198Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"ab6229c6-4e26-4495-bd24-f332c7160395","head Pod":"fail-k8s-job-g5dmq-head-mhnw8","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-g5dmq-head-mhnw8. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:23:54.198Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"ab6229c6-4e26-4495-bd24-f332c7160395","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:23:54.198Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"ab6229c6-4e26-4495-bd24-f332c7160395","worker Pod":"fail-k8s-job-g5dmq-small-group-worker-pf7h9","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-g5dmq-small-group-worker-pf7h9. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:23:54.198Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"ab6229c6-4e26-4495-bd24-f332c7160395","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:23:54.198Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"ab6229c6-4e26-4495-bd24-f332c7160395","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:23:54.198Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"ab6229c6-4e26-4495-bd24-f332c7160395","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:23:54.198Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"ab6229c6-4e26-4495-bd24-f332c7160395","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:23:54.198Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"ab6229c6-4e26-4495-bd24-f332c7160395","podName":"fail-k8s-job-g5dmq-head-mhnw8","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:22:24Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:23:54.198Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"ab6229c6-4e26-4495-bd24-f332c7160395","podName":"fail-k8s-job-g5dmq-head-mhnw8","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:22:24Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:23:54.198Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"ab6229c6-4e26-4495-bd24-f332c7160395","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:23:54.198Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"ab6229c6-4e26-4495-bd24-f332c7160395","seconds":300}
{"level":"info","ts":"2026-11-03T15:23:55.126Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"1b217110-d703-4ec6-a8d6-c467432461f1"}
{"level":"info","ts":"2026-11-03T15:23:55.126Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"1b217110-d703-4ec6-a8d6-c467432461f1"}
{"level":"info","ts":"2026-11-03T15:23:55.126Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"1b217110-d703-4ec6-a8d6-c467432461f1","1 head service found":"fail-k8s-job-g5dmq-head-svc"}
{"level":"info","ts":"2026-11-03T15:23:55.126Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"1b217110-d703-4ec6-a8d6-c467432461f1","Found 1 head Pod":"fail-k8s-job-g5dmq-head-mhnw8","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:23:55.126Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"1b217110-d703-4ec6-a8d6-c467432461f1","head Pod":"fail-k8s-job-g5dmq-head-mhnw8","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-g5dmq-head-mhnw8. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:23:55.126Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"1b217110-d703-4ec6-a8d6-c467432461f1","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:23:55.126Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"1b217110-d703-4ec6-a8d6-c467432461f1","worker Pod":"fail-k8s-job-g5dmq-small-group-worker-pf7h9","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-g5dmq-small-group-worker-pf7h9. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:23:55.126Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"1b217110-d703-4ec6-a8d6-c467432461f1","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:23:55.126Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"1b217110-d703-4ec6-a8d6-c467432461f1","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:23:55.126Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"1b217110-d703-4ec6-a8d6-c467432461f1","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:23:55.126Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"1b217110-d703-4ec6-a8d6-c467432461f1","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:23:55.126Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"1b217110-d703-4ec6-a8d6-c467432461f1","podName":"fail-k8s-job-g5dmq-head-mhnw8","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:22:24Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:23:55.126Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"1b217110-d703-4ec6-a8d6-c467432461f1","podName":"fail-k8s-job-g5dmq-head-mhnw8","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:22:24Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:23:55.126Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"1b217110-d703-4ec6-a8d6-c467432461f1","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:23:55.126Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"1b217110-d703-4ec6-a8d6-c467432461f1","seconds":300}
{"level":"info","ts":"2026-11-03T15:23:55.261Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"11b68ade-5076-407d-936f-d1c20492d52f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:23:55.261Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"11b68ade-5076-407d-936f-d1c20492d52f","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:23:55.261Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"11b68ade-5076-407d-936f-d1c20492d52f","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:23:55.261Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"11b68ade-5076-407d-936f-d1c20492d52f"}
{"level":"info","ts":"2026-11-03T15:23:55.261Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"11b68ade-5076-407d-936f-d1c20492d52f","RayCluster":"fail-k8s-job-g5dmq","State":""}
{"level":"info","ts":"2026-11-03T15:23:55.261Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"11b68ade-5076-407d-936f-d1c20492d52f","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:23:52Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:23:52Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:23:56.190Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"c49ee239-ae60-40db-bce3-5c68355578b5"}
{"level":"info","ts":"2026-11-03T15:23:56.190Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"c49ee239-ae60-40db-bce3-5c68355578b5"}
{"level":"info","ts":"2026-11-03T15:23:56.190Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"c49ee239-ae60-40db-bce3-5c68355578b5","1 head service found":"fail-k8s-job-g5dmq-head-svc"}
{"level":"info","ts":"2026-11-03T15:23:56.190Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"c49ee239-ae60-40db-bce3-5c68355578b5","Found 1 head Pod":"fail-k8s-job-g5dmq-head-mhnw8","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:23:56.190Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"c49ee239-ae60-40db-bce3-5c68355578b5","head Pod":"fail-k8s-job-g5dmq-head-mhnw8","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-g5dmq-head-mhnw8. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:23:56.190Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"c49ee239-ae60-40db-bce3-5c68355578b5","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:23:56.190Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"c49ee239-ae60-40db-bce3-5c68355578b5","worker Pod":"fail-k8s-job-g5dmq-small-group-worker-pf7h9","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-g5dmq-small-group-worker-pf7h9. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:23:56.190Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"c49ee239-ae60-40db-bce3-5c68355578b5","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:23:56.190Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"c49ee239-ae60-40db-bce3-5c68355578b5","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:23:56.190Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"c49ee239-ae60-40db-bce3-5c68355578b5","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:23:56.190Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"c49ee239-ae60-40db-bce3-5c68355578b5","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:23:56.191Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"c49ee239-ae60-40db-bce3-5c68355578b5","podName":"fail-k8s-job-g5dmq-head-mhnw8","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:22:24Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:23:56.191Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"c49ee239-ae60-40db-bce3-5c68355578b5","podName":"fail-k8s-job-g5dmq-head-mhnw8","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:22:24Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:23:56.191Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"c49ee239-ae60-40db-bce3-5c68355578b5","name":"fail-k8s-job-g5dmq","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:23:52Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:23:56Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:23:56.254Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"80644242-d9d8-47ac-a448-19dead89e482","cluster":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:23:56.255Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"4d826434-1ab3-46d2-b7d8-adb99fe7ac72","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:23:56.255Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"4d826434-1ab3-46d2-b7d8-adb99fe7ac72","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:23:56.255Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"4d826434-1ab3-46d2-b7d8-adb99fe7ac72","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:23:56.255Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"4d826434-1ab3-46d2-b7d8-adb99fe7ac72"}
{"level":"info","ts":"2026-11-03T15:23:56.255Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"4d826434-1ab3-46d2-b7d8-adb99fe7ac72","RayCluster":"fail-k8s-job-g5dmq","State":""}
{"level":"info","ts":"2026-11-03T15:23:56.255Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"4d826434-1ab3-46d2-b7d8-adb99fe7ac72","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:23:52Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:23:56Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:23:56.255Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"4d826434-1ab3-46d2-b7d8-adb99fe7ac72","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:23:56.317Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"226f3918-f001-4187-b941-cd6ebe2904d3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:23:56.317Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"226f3918-f001-4187-b941-cd6ebe2904d3","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:23:56.317Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"226f3918-f001-4187-b941-cd6ebe2904d3","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:23:56.317Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"226f3918-f001-4187-b941-cd6ebe2904d3"}
{"level":"info","ts":"2026-11-03T15:23:56.317Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"226f3918-f001-4187-b941-cd6ebe2904d3","RayCluster":"fail-k8s-job-g5dmq","State":""}
{"level":"info","ts":"2026-11-03T15:23:56.317Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"226f3918-f001-4187-b941-cd6ebe2904d3","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:23:56Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:23:56Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:23:58.255Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"04833b5c-4d23-44a8-bec3-142b08cc1d30"}
{"level":"info","ts":"2026-11-03T15:23:58.255Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"04833b5c-4d23-44a8-bec3-142b08cc1d30"}
{"level":"info","ts":"2026-11-03T15:23:58.255Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"04833b5c-4d23-44a8-bec3-142b08cc1d30","1 head service found":"fail-k8s-job-g5dmq-head-svc"}
{"level":"info","ts":"2026-11-03T15:23:58.255Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"04833b5c-4d23-44a8-bec3-142b08cc1d30","Found 1 head Pod":"fail-k8s-job-g5dmq-head-mhnw8","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:23:58.255Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"04833b5c-4d23-44a8-bec3-142b08cc1d30","head Pod":"fail-k8s-job-g5dmq-head-mhnw8","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-g5dmq-head-mhnw8. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:23:58.255Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"04833b5c-4d23-44a8-bec3-142b08cc1d30","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:23:58.255Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"04833b5c-4d23-44a8-bec3-142b08cc1d30","worker Pod":"fail-k8s-job-g5dmq-small-group-worker-pf7h9","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-g5dmq-small-group-worker-pf7h9. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:23:58.255Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"04833b5c-4d23-44a8-bec3-142b08cc1d30","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:23:58.255Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"04833b5c-4d23-44a8-bec3-142b08cc1d30","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:23:58.255Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"04833b5c-4d23-44a8-bec3-142b08cc1d30","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:23:58.255Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"04833b5c-4d23-44a8-bec3-142b08cc1d30","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:23:58.255Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"04833b5c-4d23-44a8-bec3-142b08cc1d30","podName":"fail-k8s-job-g5dmq-head-mhnw8","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:22:24Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:23:58.255Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"04833b5c-4d23-44a8-bec3-142b08cc1d30","podName":"fail-k8s-job-g5dmq-head-mhnw8","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:22:24Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:23:58.255Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"04833b5c-4d23-44a8-bec3-142b08cc1d30","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:23:58.255Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"04833b5c-4d23-44a8-bec3-142b08cc1d30","seconds":300}
{"level":"info","ts":"2026-11-03T15:23:59.318Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"b08d92a7-398f-4790-8b15-7864073a09df","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:23:59.318Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"b08d92a7-398f-4790-8b15-7864073a09df","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:23:59.318Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"b08d92a7-398f-4790-8b15-7864073a09df","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:23:59.319Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"b08d92a7-398f-4790-8b15-7864073a09df"}
{"level":"info","ts":"2026-11-03T15:23:59.319Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"b08d92a7-398f-4790-8b15-7864073a09df","RayCluster":"fail-k8s-job-g5dmq","State":""}
{"level":"info","ts":"2026-11-03T15:23:59.319Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"b08d92a7-398f-4790-8b15-7864073a09df","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:23:56Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:23:56Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:24:02.319Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"518717ab-eaec-4163-88e6-94bf5dec4b18","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:24:02.319Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"518717ab-eaec-4163-88e6-94bf5dec4b18","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:24:02.319Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"518717ab-eaec-4163-88e6-94bf5dec4b18","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:24:02.319Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"518717ab-eaec-4163-88e6-94bf5dec4b18"}
{"level":"info","ts":"2026-11-03T15:24:02.319Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"518717ab-eaec-4163-88e6-94bf5dec4b18","RayCluster":"fail-k8s-job-g5dmq","State":""}
{"level":"info","ts":"2026-11-03T15:24:02.319Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"518717ab-eaec-4163-88e6-94bf5dec4b18","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:23:56Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:23:56Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:24:03.129Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"e763b892-493d-4b19-96dd-bdad69deb272"}
{"level":"info","ts":"2026-11-03T15:24:03.129Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"e763b892-493d-4b19-96dd-bdad69deb272"}
{"level":"info","ts":"2026-11-03T15:24:03.129Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"e763b892-493d-4b19-96dd-bdad69deb272","1 head service found":"fail-k8s-job-g5dmq-head-svc"}
{"level":"info","ts":"2026-11-03T15:24:03.129Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"e763b892-493d-4b19-96dd-bdad69deb272","Found 1 head Pod":"fail-k8s-job-g5dmq-head-mhnw8","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:24:03.129Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"e763b892-493d-4b19-96dd-bdad69deb272","head Pod":"fail-k8s-job-g5dmq-head-mhnw8","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-g5dmq-head-mhnw8. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:24:03.129Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"e763b892-493d-4b19-96dd-bdad69deb272","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:24:03.129Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"e763b892-493d-4b19-96dd-bdad69deb272","worker Pod":"fail-k8s-job-g5dmq-small-group-worker-pf7h9","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-g5dmq-small-group-worker-pf7h9. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:24:03.129Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"e763b892-493d-4b19-96dd-bdad69deb272","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:24:03.129Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"e763b892-493d-4b19-96dd-bdad69deb272","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:24:03.129Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"e763b892-493d-4b19-96dd-bdad69deb272","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:24:03.129Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"e763b892-493d-4b19-96dd-bdad69deb272","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:24:03.129Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"e763b892-493d-4b19-96dd-bdad69deb272","podName":"fail-k8s-job-g5dmq-small-group-worker-pf7h9","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:22:24Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:24:03.129Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"e763b892-493d-4b19-96dd-bdad69deb272","podName":"fail-k8s-job-g5dmq-small-group-worker-pf7h9","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:22:24Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:24:03.130Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"e763b892-493d-4b19-96dd-bdad69deb272","name":"fail-k8s-job-g5dmq","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:23:56Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:24:03Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:24:03.192Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"b16bf16f-5708-40a0-b0f9-c807c91c4ce9","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:24:03.192Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"b16bf16f-5708-40a0-b0f9-c807c91c4ce9","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:24:03.192Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"b16bf16f-5708-40a0-b0f9-c807c91c4ce9","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:24:03.192Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"815d0e9a-64a1-44cf-bf16-dcdc6ff20444","cluster":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:24:03.192Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"b16bf16f-5708-40a0-b0f9-c807c91c4ce9"}
{"level":"info","ts":"2026-11-03T15:24:03.192Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"b16bf16f-5708-40a0-b0f9-c807c91c4ce9","RayCluster":"fail-k8s-job-g5dmq","State":""}
{"level":"info","ts":"2026-11-03T15:24:03.192Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"b16bf16f-5708-40a0-b0f9-c807c91c4ce9","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:23:56Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:24:03Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:24:03.192Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"b16bf16f-5708-40a0-b0f9-c807c91c4ce9","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:24:03.253Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"3f4c8e49-aa73-4de6-a839-e205f60303fc","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:24:03.253Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"3f4c8e49-aa73-4de6-a839-e205f60303fc","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:24:03.253Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"3f4c8e49-aa73-4de6-a839-e205f60303fc","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:24:03.253Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"3f4c8e49-aa73-4de6-a839-e205f60303fc"}
{"level":"info","ts":"2026-11-03T15:24:03.253Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"3f4c8e49-aa73-4de6-a839-e205f60303fc","RayCluster":"fail-k8s-job-g5dmq","State":""}
{"level":"info","ts":"2026-11-03T15:24:03.253Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"3f4c8e49-aa73-4de6-a839-e205f60303fc","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:24:03Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:24:03Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:24:05.192Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"f911a833-bb47-432a-be75-07286f8fcd7a"}
{"level":"info","ts":"2026-11-03T15:24:05.192Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"f911a833-bb47-432a-be75-07286f8fcd7a"}
{"level":"info","ts":"2026-11-03T15:24:05.192Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"f911a833-bb47-432a-be75-07286f8fcd7a","1 head service found":"fail-k8s-job-g5dmq-head-svc"}
{"level":"info","ts":"2026-11-03T15:24:05.193Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"f911a833-bb47-432a-be75-07286f8fcd7a","Found 1 head Pod":"fail-k8s-job-g5dmq-head-mhnw8","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:24:05.193Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"f911a833-bb47-432a-be75-07286f8fcd7a","head Pod":"fail-k8s-job-g5dmq-head-mhnw8","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-g5dmq-head-mhnw8. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:24:05.193Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"f911a833-bb47-432a-be75-07286f8fcd7a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:24:05.193Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"f911a833-bb47-432a-be75-07286f8fcd7a","worker Pod":"fail-k8s-job-g5dmq-small-group-worker-pf7h9","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-g5dmq-small-group-worker-pf7h9. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:24:05.193Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"f911a833-bb47-432a-be75-07286f8fcd7a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:24:05.193Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"f911a833-bb47-432a-be75-07286f8fcd7a","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:24:05.193Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"f911a833-bb47-432a-be75-07286f8fcd7a","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:24:05.193Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"f911a833-bb47-432a-be75-07286f8fcd7a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:24:05.193Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"f911a833-bb47-432a-be75-07286f8fcd7a","podName":"fail-k8s-job-g5dmq-small-group-worker-pf7h9","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:22:24Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:24:05.193Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"f911a833-bb47-432a-be75-07286f8fcd7a","podName":"fail-k8s-job-g5dmq-small-group-worker-pf7h9","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:22:24Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:24:05.193Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"f911a833-bb47-432a-be75-07286f8fcd7a","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:24:05.193Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"f911a833-bb47-432a-be75-07286f8fcd7a","seconds":300}
{"level":"info","ts":"2026-11-03T15:24:06.254Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"0f4b69b7-583b-489b-bf57-3f564817d86e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:24:06.254Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"0f4b69b7-583b-489b-bf57-3f564817d86e","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:24:06.254Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"0f4b69b7-583b-489b-bf57-3f564817d86e","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:24:06.254Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"0f4b69b7-583b-489b-bf57-3f564817d86e"}
{"level":"info","ts":"2026-11-03T15:24:06.254Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"0f4b69b7-583b-489b-bf57-3f564817d86e","RayCluster":"fail-k8s-job-g5dmq","State":""}
{"level":"info","ts":"2026-11-03T15:24:06.254Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"0f4b69b7-583b-489b-bf57-3f564817d86e","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:24:03Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:24:03Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:24:07.158Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"f994e7b2-93b8-4e10-b3e7-db5f353c5d84"}
{"level":"info","ts":"2026-11-03T15:24:07.159Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"f994e7b2-93b8-4e10-b3e7-db5f353c5d84"}
{"level":"info","ts":"2026-11-03T15:24:07.159Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"f994e7b2-93b8-4e10-b3e7-db5f353c5d84","1 head service found":"fail-k8s-job-g5dmq-head-svc"}
{"level":"info","ts":"2026-11-03T15:24:07.159Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"f994e7b2-93b8-4e10-b3e7-db5f353c5d84","Found 1 head Pod":"fail-k8s-job-g5dmq-head-mhnw8","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:24:07.159Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"f994e7b2-93b8-4e10-b3e7-db5f353c5d84","head Pod":"fail-k8s-job-g5dmq-head-mhnw8","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-g5dmq-head-mhnw8. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:24:07.159Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"f994e7b2-93b8-4e10-b3e7-db5f353c5d84","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:24:07.159Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"f994e7b2-93b8-4e10-b3e7-db5f353c5d84","worker Pod":"fail-k8s-job-g5dmq-small-group-worker-pf7h9","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-g5dmq-small-group-worker-pf7h9. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:24:07.159Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"f994e7b2-93b8-4e10-b3e7-db5f353c5d84","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:24:07.159Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"f994e7b2-93b8-4e10-b3e7-db5f353c5d84","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:24:07.159Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"f994e7b2-93b8-4e10-b3e7-db5f353c5d84","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:24:07.159Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"f994e7b2-93b8-4e10-b3e7-db5f353c5d84","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:24:07.159Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"f994e7b2-93b8-4e10-b3e7-db5f353c5d84","name":"fail-k8s-job-g5dmq","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:24:03Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:07Z","stateTransitionTimes":{"ready":"2026-11-03T15:24:07Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:24:03Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:24:07Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:24:07.222Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"252d7dd5-4b39-475c-a14e-9b2b3283a458","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:24:07.222Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"252d7dd5-4b39-475c-a14e-9b2b3283a458","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:24:07.222Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"345b47d9-7bc8-4542-bbdd-f3495012a8dc","cluster":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:24:07.222Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"252d7dd5-4b39-475c-a14e-9b2b3283a458","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:24:07.222Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"252d7dd5-4b39-475c-a14e-9b2b3283a458"}
{"level":"info","ts":"2026-11-03T15:24:07.289Z","logger":"controllers.RayJob","msg":"Created submitter Kubernetes Job for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"252d7dd5-4b39-475c-a14e-9b2b3283a458","Kubernetes Job":"fail-k8s-job"}
{"level":"info","ts":"2026-11-03T15:24:07.289Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"252d7dd5-4b39-475c-a14e-9b2b3283a458","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:24:03Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","dashboardURL":"fail-k8s-job-g5dmq-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:24:03Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:24:07.290Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"252d7dd5-4b39-475c-a14e-9b2b3283a458","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-11-03T15:24:07.351Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"e473d866-eff5-44e8-85e9-c9752264c230","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:24:07.351Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"e473d866-eff5-44e8-85e9-c9752264c230","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:24:07.351Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"e473d866-eff5-44e8-85e9-c9752264c230","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:24:07.352Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"e473d866-eff5-44e8-85e9-c9752264c230"}
{"level":"info","ts":"2026-11-03T15:24:07.352Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job for RayJob already exists","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"e473d866-eff5-44e8-85e9-c9752264c230","Kubernetes Job":"fail-k8s-job"}
{"level":"info","ts":"2026-11-03T15:24:07.352Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"e473d866-eff5-44e8-85e9-c9752264c230","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:24:03Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","dashboardURL":"fail-k8s-job-g5dmq-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:24:03Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:24:07.352Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"e473d866-eff5-44e8-85e9-c9752264c230","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-11-03T15:24:07.407Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"e473d866-eff5-44e8-85e9-c9752264c230","error":"Operation cannot be fulfilled on rayjobs.ray.io \"fail-k8s-job\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-11-03T15:24:07.408Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"e473d866-eff5-44e8-85e9-c9752264c230"}
{"level":"error","ts":"2026-11-03T15:24:07.408Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"e473d866-eff5-44e8-85e9-c9752264c230","error":"Operation cannot be fulfilled on rayjobs.ray.io \"fail-k8s-job\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:24:07.408Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"213a0799-5586-4ff2-8d48-3d09da373643","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:24:07.408Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"213a0799-5586-4ff2-8d48-3d09da373643","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:24:07.408Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"213a0799-5586-4ff2-8d48-3d09da373643","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:24:07.408Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"213a0799-5586-4ff2-8d48-3d09da373643"}
{"level":"info","ts":"2026-11-03T15:24:07.408Z","logger":"controllers.RayJob","msg":"Create a Kubernetes Service","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"213a0799-5586-4ff2-8d48-3d09da373643","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:24:07.471Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"76a9be22-b253-4090-a3dd-0f90afabb134","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:24:07.471Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"76a9be22-b253-4090-a3dd-0f90afabb134","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:24:07.471Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"76a9be22-b253-4090-a3dd-0f90afabb134","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:24:07.471Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"76a9be22-b253-4090-a3dd-0f90afabb134"}
{"level":"info","ts":"2026-11-03T15:24:07.472Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"76a9be22-b253-4090-a3dd-0f90afabb134","rayCluster":"fail-k8s-job-g5dmq","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:24:09.225Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"ef2a9b8f-4882-4548-98ad-08562e0b996a"}
{"level":"info","ts":"2026-11-03T15:24:09.225Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"ef2a9b8f-4882-4548-98ad-08562e0b996a"}
{"level":"info","ts":"2026-11-03T15:24:09.225Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"ef2a9b8f-4882-4548-98ad-08562e0b996a","1 head service found":"fail-k8s-job-g5dmq-head-svc"}
{"level":"info","ts":"2026-11-03T15:24:09.225Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"ef2a9b8f-4882-4548-98ad-08562e0b996a","Found 1 head Pod":"fail-k8s-job-g5dmq-head-mhnw8","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:24:09.225Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"ef2a9b8f-4882-4548-98ad-08562e0b996a","head Pod":"fail-k8s-job-g5dmq-head-mhnw8","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod fail-k8s-job-g5dmq-head-mhnw8. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:24:09.225Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"ef2a9b8f-4882-4548-98ad-08562e0b996a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:24:09.225Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"ef2a9b8f-4882-4548-98ad-08562e0b996a","worker Pod":"fail-k8s-job-g5dmq-small-group-worker-pf7h9","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod fail-k8s-job-g5dmq-small-group-worker-pf7h9. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:24:09.225Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"ef2a9b8f-4882-4548-98ad-08562e0b996a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:24:09.225Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"ef2a9b8f-4882-4548-98ad-08562e0b996a","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:24:09.225Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"ef2a9b8f-4882-4548-98ad-08562e0b996a","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:24:09.225Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"ef2a9b8f-4882-4548-98ad-08562e0b996a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:24:09.225Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"ef2a9b8f-4882-4548-98ad-08562e0b996a","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:24:09.225Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"ef2a9b8f-4882-4548-98ad-08562e0b996a","seconds":300}
{"level":"info","ts":"2026-11-03T15:24:10.144Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"f22e3dd8-5219-4803-b8f2-4cce5f09d27d","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:24:10.144Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"f22e3dd8-5219-4803-b8f2-4cce5f09d27d","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:24:10.144Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"f22e3dd8-5219-4803-b8f2-4cce5f09d27d","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:24:10.144Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"f22e3dd8-5219-4803-b8f2-4cce5f09d27d"}
{"level":"info","ts":"2026-11-03T15:24:10.144Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"f22e3dd8-5219-4803-b8f2-4cce5f09d27d","rayCluster":"fail-k8s-job-g5dmq","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:24:13.151Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"2e447449-c108-4bcb-9e6b-1ef3917c74de","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:24:13.151Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"2e447449-c108-4bcb-9e6b-1ef3917c74de","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:24:13.151Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"2e447449-c108-4bcb-9e6b-1ef3917c74de","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:24:13.151Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"2e447449-c108-4bcb-9e6b-1ef3917c74de"}
{"level":"info","ts":"2026-11-03T15:24:13.151Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"2e447449-c108-4bcb-9e6b-1ef3917c74de","rayCluster":"fail-k8s-job-g5dmq","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:24:13.167Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"8431633f-2403-4c6a-82d1-0c69c24a8239","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:24:13.167Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"8431633f-2403-4c6a-82d1-0c69c24a8239","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:24:13.167Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"8431633f-2403-4c6a-82d1-0c69c24a8239","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:24:13.167Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"8431633f-2403-4c6a-82d1-0c69c24a8239"}
{"level":"info","ts":"2026-11-03T15:24:13.167Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"8431633f-2403-4c6a-82d1-0c69c24a8239","rayCluster":"fail-k8s-job-g5dmq","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:24:14.173Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"db2f7bcb-21c0-4a62-917c-dc2bd0851118","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:24:14.173Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"db2f7bcb-21c0-4a62-917c-dc2bd0851118","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:24:14.173Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"db2f7bcb-21c0-4a62-917c-dc2bd0851118","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:24:14.173Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"db2f7bcb-21c0-4a62-917c-dc2bd0851118"}
{"level":"info","ts":"2026-11-03T15:24:14.173Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"db2f7bcb-21c0-4a62-917c-dc2bd0851118","rayCluster":"fail-k8s-job-g5dmq","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:24:14.198Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"42c70e30-8279-44a9-812a-40ef0bdf290d","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:24:14.198Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"42c70e30-8279-44a9-812a-40ef0bdf290d","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:24:14.198Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"42c70e30-8279-44a9-812a-40ef0bdf290d","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:24:14.198Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"42c70e30-8279-44a9-812a-40ef0bdf290d"}
{"level":"info","ts":"2026-11-03T15:24:14.198Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"42c70e30-8279-44a9-812a-40ef0bdf290d","rayCluster":"fail-k8s-job-g5dmq","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:24:17.203Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"23661b01-a8cd-42b5-bac4-3e8808324786","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:24:17.203Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"23661b01-a8cd-42b5-bac4-3e8808324786","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:24:17.204Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"23661b01-a8cd-42b5-bac4-3e8808324786","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:24:17.204Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"23661b01-a8cd-42b5-bac4-3e8808324786"}
{"level":"info","ts":"2026-11-03T15:24:17.204Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"23661b01-a8cd-42b5-bac4-3e8808324786","rayCluster":"fail-k8s-job-g5dmq","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:24:20.211Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"bae3eb25-00e1-470a-b7ce-1e904433733c","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:24:20.211Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"bae3eb25-00e1-470a-b7ce-1e904433733c","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:24:20.211Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"bae3eb25-00e1-470a-b7ce-1e904433733c","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:24:20.211Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"bae3eb25-00e1-470a-b7ce-1e904433733c"}
{"level":"info","ts":"2026-11-03T15:24:20.211Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"bae3eb25-00e1-470a-b7ce-1e904433733c","rayCluster":"fail-k8s-job-g5dmq","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:24:21.050Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"d8a73a23-ff0c-4cbe-98aa-4f4192289b8e","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:24:21.050Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"d8a73a23-ff0c-4cbe-98aa-4f4192289b8e","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:24:21.050Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"d8a73a23-ff0c-4cbe-98aa-4f4192289b8e","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:24:21.050Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"d8a73a23-ff0c-4cbe-98aa-4f4192289b8e"}
{"level":"info","ts":"2026-11-03T15:24:21.050Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"d8a73a23-ff0c-4cbe-98aa-4f4192289b8e","rayCluster":"fail-k8s-job-g5dmq","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:24:24.057Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"8bb5929b-dfde-4b19-9cf4-416149ee464d","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:24:24.057Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"8bb5929b-dfde-4b19-9cf4-416149ee464d","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:24:24.057Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"8bb5929b-dfde-4b19-9cf4-416149ee464d","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:24:24.057Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"8bb5929b-dfde-4b19-9cf4-416149ee464d"}
{"level":"info","ts":"2026-11-03T15:24:24.058Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"8bb5929b-dfde-4b19-9cf4-416149ee464d","rayCluster":"fail-k8s-job-g5dmq","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:24:24.205Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"01f6ac08-73df-45ad-9784-7f8018817381","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:24:24.205Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"01f6ac08-73df-45ad-9784-7f8018817381","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:24:24.205Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"01f6ac08-73df-45ad-9784-7f8018817381","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:24:24.205Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"01f6ac08-73df-45ad-9784-7f8018817381"}
{"level":"info","ts":"2026-11-03T15:24:24.205Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"01f6ac08-73df-45ad-9784-7f8018817381","rayCluster":"fail-k8s-job-g5dmq","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:24:27.211Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"1a60f410-589c-413e-ab90-e48f2f35129e","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:24:27.211Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"1a60f410-589c-413e-ab90-e48f2f35129e","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:24:27.211Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"1a60f410-589c-413e-ab90-e48f2f35129e","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:24:27.212Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"1a60f410-589c-413e-ab90-e48f2f35129e"}
{"level":"info","ts":"2026-11-03T15:24:27.212Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"1a60f410-589c-413e-ab90-e48f2f35129e","rayCluster":"fail-k8s-job-g5dmq","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:24:27.225Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"a9659057-4ad5-4fa5-b267-0f98031d7d48","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:24:27.225Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"a9659057-4ad5-4fa5-b267-0f98031d7d48","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:24:27.225Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"a9659057-4ad5-4fa5-b267-0f98031d7d48","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:24:27.225Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"a9659057-4ad5-4fa5-b267-0f98031d7d48"}
{"level":"info","ts":"2026-11-03T15:24:27.225Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"a9659057-4ad5-4fa5-b267-0f98031d7d48","rayCluster":"fail-k8s-job-g5dmq","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:24:28.233Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"c26c799d-d2c6-4630-9359-591f5c8ff856","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:24:28.233Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"c26c799d-d2c6-4630-9359-591f5c8ff856","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:24:28.233Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"c26c799d-d2c6-4630-9359-591f5c8ff856","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:24:28.233Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"c26c799d-d2c6-4630-9359-591f5c8ff856"}
{"level":"info","ts":"2026-11-03T15:24:28.233Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"c26c799d-d2c6-4630-9359-591f5c8ff856","rayCluster":"fail-k8s-job-g5dmq","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:24:28.253Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"cab1f2ae-96ec-468b-b20e-de97350b5096","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:24:28.253Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"cab1f2ae-96ec-468b-b20e-de97350b5096","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:24:28.253Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"cab1f2ae-96ec-468b-b20e-de97350b5096","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:24:28.253Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"cab1f2ae-96ec-468b-b20e-de97350b5096"}
{"level":"info","ts":"2026-11-03T15:24:28.253Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"cab1f2ae-96ec-468b-b20e-de97350b5096","rayCluster":"fail-k8s-job-g5dmq","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:24:31.259Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"20f772e5-439d-40d7-9bed-172ce5d766c1","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:24:31.259Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"20f772e5-439d-40d7-9bed-172ce5d766c1","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:24:31.259Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"20f772e5-439d-40d7-9bed-172ce5d766c1","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:24:31.259Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"20f772e5-439d-40d7-9bed-172ce5d766c1"}
{"level":"info","ts":"2026-11-03T15:24:31.259Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"20f772e5-439d-40d7-9bed-172ce5d766c1","rayCluster":"fail-k8s-job-g5dmq","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:24:34.265Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"93db8142-a48b-471a-adb9-ea0a45aa1a5c","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:24:34.265Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"93db8142-a48b-471a-adb9-ea0a45aa1a5c","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:24:34.265Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"93db8142-a48b-471a-adb9-ea0a45aa1a5c","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:24:34.266Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"93db8142-a48b-471a-adb9-ea0a45aa1a5c"}
{"level":"info","ts":"2026-11-03T15:24:34.266Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"93db8142-a48b-471a-adb9-ea0a45aa1a5c","rayCluster":"fail-k8s-job-g5dmq","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:24:37.272Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"c321f457-2552-4117-8ae9-4bee6c18c004","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:24:37.272Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"c321f457-2552-4117-8ae9-4bee6c18c004","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:24:37.272Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"c321f457-2552-4117-8ae9-4bee6c18c004","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:24:37.272Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"c321f457-2552-4117-8ae9-4bee6c18c004"}
{"level":"info","ts":"2026-11-03T15:24:37.272Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"c321f457-2552-4117-8ae9-4bee6c18c004","rayCluster":"fail-k8s-job-g5dmq","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:24:40.279Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"e7001bca-46fc-4f38-9551-bed34a741a5c","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:24:40.279Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"e7001bca-46fc-4f38-9551-bed34a741a5c","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:24:40.279Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"e7001bca-46fc-4f38-9551-bed34a741a5c","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:24:40.279Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"e7001bca-46fc-4f38-9551-bed34a741a5c"}
{"level":"info","ts":"2026-11-03T15:24:40.279Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"e7001bca-46fc-4f38-9551-bed34a741a5c","rayCluster":"fail-k8s-job-g5dmq","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:24:43.286Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"f1e55937-58b5-4573-84e4-dc13b3fd94a3","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:24:43.286Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"f1e55937-58b5-4573-84e4-dc13b3fd94a3","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:24:43.286Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"f1e55937-58b5-4573-84e4-dc13b3fd94a3","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:24:43.286Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"f1e55937-58b5-4573-84e4-dc13b3fd94a3"}
{"level":"info","ts":"2026-11-03T15:24:43.286Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"f1e55937-58b5-4573-84e4-dc13b3fd94a3","rayCluster":"fail-k8s-job-g5dmq","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:24:45.048Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"6e34e19c-f8b3-4fde-979e-6ff6f96e367e","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:24:45.048Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"6e34e19c-f8b3-4fde-979e-6ff6f96e367e","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:24:45.048Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"6e34e19c-f8b3-4fde-979e-6ff6f96e367e","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:24:45.048Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"6e34e19c-f8b3-4fde-979e-6ff6f96e367e"}
{"level":"info","ts":"2026-11-03T15:24:45.048Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"6e34e19c-f8b3-4fde-979e-6ff6f96e367e","rayCluster":"fail-k8s-job-g5dmq","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:24:47.312Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"7550cc5e-cfdc-48a4-873b-8765d405471f","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:24:47.312Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"7550cc5e-cfdc-48a4-873b-8765d405471f","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:24:47.312Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"7550cc5e-cfdc-48a4-873b-8765d405471f","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:24:47.312Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"7550cc5e-cfdc-48a4-873b-8765d405471f"}
{"level":"info","ts":"2026-11-03T15:24:47.312Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"7550cc5e-cfdc-48a4-873b-8765d405471f","rayCluster":"fail-k8s-job-g5dmq","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:24:50.318Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"51c1eaf4-7fd4-4df9-885d-2fe4208720b1","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:24:50.319Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"51c1eaf4-7fd4-4df9-885d-2fe4208720b1","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:24:50.319Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"51c1eaf4-7fd4-4df9-885d-2fe4208720b1","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:24:50.319Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"51c1eaf4-7fd4-4df9-885d-2fe4208720b1"}
{"level":"info","ts":"2026-11-03T15:24:50.319Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"51c1eaf4-7fd4-4df9-885d-2fe4208720b1","rayCluster":"fail-k8s-job-g5dmq","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:24:51.326Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"827636a9-60cf-4dc0-9a8d-01f44745e1f0","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:24:51.326Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"827636a9-60cf-4dc0-9a8d-01f44745e1f0","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:24:51.326Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"827636a9-60cf-4dc0-9a8d-01f44745e1f0","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:24:51.327Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"827636a9-60cf-4dc0-9a8d-01f44745e1f0"}
{"level":"info","ts":"2026-11-03T15:24:51.327Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"827636a9-60cf-4dc0-9a8d-01f44745e1f0","rayCluster":"fail-k8s-job-g5dmq","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:24:52.335Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"69c9f754-7458-49a1-8b42-0aa7e6d168e5","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:24:52.335Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"69c9f754-7458-49a1-8b42-0aa7e6d168e5","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:24:52.335Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"69c9f754-7458-49a1-8b42-0aa7e6d168e5","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:24:52.335Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"69c9f754-7458-49a1-8b42-0aa7e6d168e5"}
{"level":"info","ts":"2026-11-03T15:24:52.335Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"69c9f754-7458-49a1-8b42-0aa7e6d168e5","rayCluster":"fail-k8s-job-g5dmq","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:24:52.354Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"3675ad96-0d15-422b-a454-4a5578b7ccb4","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:24:52.354Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"3675ad96-0d15-422b-a454-4a5578b7ccb4","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:24:52.354Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"3675ad96-0d15-422b-a454-4a5578b7ccb4","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:24:52.354Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"3675ad96-0d15-422b-a454-4a5578b7ccb4"}
{"level":"info","ts":"2026-11-03T15:24:52.354Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"3675ad96-0d15-422b-a454-4a5578b7ccb4","rayCluster":"fail-k8s-job-g5dmq","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:24:52.354Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job has failed. Attempting to transition the status to `Failed`.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"3675ad96-0d15-422b-a454-4a5578b7ccb4","Submitter K8s Job":"fail-k8s-job","Reason":"BackoffLimitExceeded","Message":"Job has reached the specified backoff limit"}
{"level":"info","ts":"2026-11-03T15:24:52.354Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"3675ad96-0d15-422b-a454-4a5578b7ccb4","oldRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","dashboardURL":"fail-k8s-job-g5dmq-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:24:03Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"fail-k8s-job-dkm4q","rayClusterName":"fail-k8s-job-g5dmq","dashboardURL":"fail-k8s-job-g5dmq-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobDeploymentStatus":"Failed","reason":"SubmissionFailed","message":"Job submission has failed. Reason: BackoffLimitExceeded. Message: Job has reached the specified backoff limit","startTime":"2026-11-03T15:22:20Z","succeeded":0,"failed":1,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:03Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.40","serviceIP":"10.132.0.40","podName":"fail-k8s-job-g5dmq-head-mhnw8","serviceName":"fail-k8s-job-g5dmq-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:24:03Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:22:20Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:22:21Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:24:52.354Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"3675ad96-0d15-422b-a454-4a5578b7ccb4","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Failed"}
{"level":"info","ts":"2026-11-03T15:24:52.452Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"c83c658d-83a7-4f4e-bbf1-433a0a0f3684","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:24:52.452Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"c83c658d-83a7-4f4e-bbf1-433a0a0f3684","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Failed","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-11-03 15:24:52 +0000 UTC","Now":"2026-11-03T15:24:52.452Z","ShutdownTime":"2026-11-03T15:24:52.000Z"}
{"level":"info","ts":"2026-11-03T15:24:52.511Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob is deleted","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"c83c658d-83a7-4f4e-bbf1-433a0a0f3684","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:24:52.511Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"c83c658d-83a7-4f4e-bbf1-433a0a0f3684","isClusterDeleted":false}
{"level":"info","ts":"2026-11-03T15:24:52.511Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"c83c658d-83a7-4f4e-bbf1-433a0a0f3684","deletionMechanism":"ShutdownAfterJobFinishes","RayCluster":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:24:52.511Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"reconcileID":"7b5c1f71-0d8b-43a4-94f2-99dcb38ac09a"}
{"level":"info","ts":"2026-11-03T15:24:52.511Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"fail-k8s-job-g5dmq","reconcileID":"0648ca4a-c638-46f5-ab98-e2e54d39e6ae","namespacedName":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:24:52.511Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"fail-k8s-job-g5dmq","reconcileID":"0648ca4a-c638-46f5-ab98-e2e54d39e6ae","cluster":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:24:52.511Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"fail-k8s-job-g5dmq","reconcileID":"0648ca4a-c638-46f5-ab98-e2e54d39e6ae","cluster":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:24:52.511Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"f63c64dd-2279-4361-be04-1f7b537fdfb0","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:24:52.511Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"f63c64dd-2279-4361-be04-1f7b537fdfb0","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Failed","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-11-03 15:24:52 +0000 UTC","Now":"2026-11-03T15:24:52.511Z","ShutdownTime":"2026-11-03T15:24:52.000Z"}
{"level":"info","ts":"2026-11-03T15:24:52.511Z","logger":"controllers.RayJob","msg":"The deletion of the associated RayCluster for RayJob is ongoing.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"f63c64dd-2279-4361-be04-1f7b537fdfb0","RayCluster":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:24:52.511Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"f63c64dd-2279-4361-be04-1f7b537fdfb0","isClusterDeleted":false}
{"level":"info","ts":"2026-11-03T15:24:52.511Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"f63c64dd-2279-4361-be04-1f7b537fdfb0","deletionMechanism":"ShutdownAfterJobFinishes","RayCluster":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:24:52.511Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"fail-k8s-job-g5dmq","reconcileID":"0648ca4a-c638-46f5-ab98-e2e54d39e6ae","namespace":"test-ns-kbx4w","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-11-03T15:24:52.565Z","logger":"authentication-controller","msg":"Deleted shared ReferenceGrant (last cluster with auth)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"fail-k8s-job-g5dmq","reconcileID":"0648ca4a-c638-46f5-ab98-e2e54d39e6ae","name":"kuberay-gateway-access","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:24:52.618Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"fail-k8s-job-g5dmq","reconcileID":"0648ca4a-c638-46f5-ab98-e2e54d39e6ae","name":"test-ns-kbx4w-fail-k8s-job-g5dmq","namespace":"opendatahub"}
{"level":"info","ts":"2026-11-03T15:24:52.670Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"fail-k8s-job-g5dmq","reconcileID":"0648ca4a-c638-46f5-ab98-e2e54d39e6ae","configMap":"kube-rbac-proxy-config-fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:24:52.722Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"fail-k8s-job-g5dmq","reconcileID":"0648ca4a-c638-46f5-ab98-e2e54d39e6ae","serviceAccount":"fail-k8s-job-g5dmq-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-11-03T15:24:52.902Z","logger":"raycluster-resource","msg":"default","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:24:52.902Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:24:53.066Z","logger":"raycluster-resource","msg":"validate update","name":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:24:53.121Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"fail-k8s-job-g5dmq","reconcileID":"0648ca4a-c638-46f5-ab98-e2e54d39e6ae","cluster":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:24:53.121Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"fail-k8s-job-g5dmq","reconcileID":"0a89d578-91fd-44bd-8bee-5afb5a13b389","namespacedName":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:24:53.121Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"fail-k8s-job-g5dmq","reconcileID":"0a89d578-91fd-44bd-8bee-5afb5a13b389","cluster":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:24:53.121Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"fail-k8s-job-g5dmq","reconcileID":"0a89d578-91fd-44bd-8bee-5afb5a13b389","cluster":"fail-k8s-job-g5dmq"}
{"level":"info","ts":"2026-11-03T15:24:53.121Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"fail-k8s-job-g5dmq","reconcileID":"0a89d578-91fd-44bd-8bee-5afb5a13b389","namespace":"test-ns-kbx4w","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-11-03T15:24:53.121Z","logger":"authentication-controller","msg":"ReferenceGrant already deleted","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"fail-k8s-job-g5dmq","reconcileID":"0a89d578-91fd-44bd-8bee-5afb5a13b389","name":"kuberay-gateway-access"}
{"level":"info","ts":"2026-11-03T15:24:53.122Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"6b825985-10ac-4570-8e98-2cbf31a3e186","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:24:53.122Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"6b825985-10ac-4570-8e98-2cbf31a3e186","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Failed","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-11-03 15:24:52 +0000 UTC","Now":"2026-11-03T15:24:53.122Z","ShutdownTime":"2026-11-03T15:24:52.000Z"}
{"level":"info","ts":"2026-11-03T15:24:53.122Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"6b825985-10ac-4570-8e98-2cbf31a3e186","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:24:53.122Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"6b825985-10ac-4570-8e98-2cbf31a3e186","isClusterDeleted":true}
{"level":"info","ts":"2026-11-03T15:24:53.122Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"6b825985-10ac-4570-8e98-2cbf31a3e186","deletionMechanism":"ShutdownAfterJobFinishes","RayCluster":"fail-k8s-job-g5dmq"}
{"level":"error","ts":"2026-11-03T15:24:53.180Z","logger":"authentication-controller","msg":"Failed to remove finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"fail-k8s-job-g5dmq","reconcileID":"0a89d578-91fd-44bd-8bee-5afb5a13b389","error":"rayclusters.ray.io \"fail-k8s-job-g5dmq\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleDeletion\n\t/workspace/controllers/ray/authentication_controller.go:195\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:24:53.180Z","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"fail-k8s-job-g5dmq","reconcileID":"0a89d578-91fd-44bd-8bee-5afb5a13b389"}
{"level":"error","ts":"2026-11-03T15:24:53.180Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"fail-k8s-job-g5dmq","reconcileID":"0a89d578-91fd-44bd-8bee-5afb5a13b389","error":"rayclusters.ray.io \"fail-k8s-job-g5dmq\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:24:53.180Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"fail-k8s-job-g5dmq","reconcileID":"1a791a69-baa2-4a02-8095-f162f6daf4e4","namespacedName":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:24:53.180Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"fail-k8s-job-g5dmq","reconcileID":"1a791a69-baa2-4a02-8095-f162f6daf4e4"}
{"level":"info","ts":"2026-11-03T15:24:53.180Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"fail-k8s-job-g5dmq","reconcileID":"1a791a69-baa2-4a02-8095-f162f6daf4e4","namespace":"test-ns-kbx4w","authClusters":0}
{"level":"info","ts":"2026-11-03T15:24:53.181Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"fail-k8s-job-g5dmq","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"fail-k8s-job-g5dmq","reconcileID":"1a791a69-baa2-4a02-8095-f162f6daf4e4","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:24:53.294Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"cd1be1cc-005d-483a-8dea-caf39f1b30fb","DeletionTimestamp":"2026-11-03 15:24:53 +0000 UTC"}
{"level":"error","ts":"2026-11-03T15:24:53.294Z","logger":"controllers.RayJob","msg":"Failed to get RayCluster","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"cd1be1cc-005d-483a-8dea-caf39f1b30fb","error":"RayCluster.ray.io \"fail-k8s-job-g5dmq\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:120\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-03T15:24:53.299Z","logger":"controllers.RayJob","msg":"Failed to stop job for RayJob","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"cd1be1cc-005d-483a-8dea-caf39f1b30fb","error":"invalid character 'J' looking for beginning of value","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:128\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:24:53.299Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"cd1be1cc-005d-483a-8dea-caf39f1b30fb","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-11-03T15:24:53.367Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"37b3e0b6-032c-4f64-873c-254a5a387f37","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:24:53.367Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"37b3e0b6-032c-4f64-873c-254a5a387f37","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-11-03T15:24:53.427Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"37b3e0b6-032c-4f64-873c-254a5a387f37"}
{"level":"info","ts":"2026-11-03T15:24:53.427Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"37b3e0b6-032c-4f64-873c-254a5a387f37","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-11-03T15:24:53.427Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"37b3e0b6-032c-4f64-873c-254a5a387f37","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:24:53.427Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"37b3e0b6-032c-4f64-873c-254a5a387f37","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:24:53.486Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"fail-k8s-job","namespace":"test-ns-kbx4w"},"reconcileID":"28f07f1a-2143-413d-943c-86d15db28887"}
{"level":"info","ts":"2026-11-03T15:24:53.486Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"60b0db8e-45e1-4fa2-82a5-e8dd1e6c5920","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:24:53.486Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"60b0db8e-45e1-4fa2-82a5-e8dd1e6c5920","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:24:53.486Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"60b0db8e-45e1-4fa2-82a5-e8dd1e6c5920","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:24:53.486Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"60b0db8e-45e1-4fa2-82a5-e8dd1e6c5920","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:24:53.541Z","logger":"raycluster-resource","msg":"default","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:24:53.541Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"stop-lwhxj","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:24:53.601Z","logger":"raycluster-resource","msg":"validate create","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:24:53.655Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"60b0db8e-45e1-4fa2-82a5-e8dd1e6c5920","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:24:53.655Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"e4722055-d5fb-41c4-a662-aea8e2a8593d","cluster":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:24:53.655Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"60b0db8e-45e1-4fa2-82a5-e8dd1e6c5920"}
{"level":"info","ts":"2026-11-03T15:24:53.655Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"60b0db8e-45e1-4fa2-82a5-e8dd1e6c5920","RayCluster":"stop-lwhxj","State":""}
{"level":"info","ts":"2026-11-03T15:24:53.655Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"60b0db8e-45e1-4fa2-82a5-e8dd1e6c5920","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:24:53.655Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"7a1a8827-c2d4-473c-8973-77ae5f2ba0b7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:24:53.655Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"7a1a8827-c2d4-473c-8973-77ae5f2ba0b7","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:24:53.655Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"7a1a8827-c2d4-473c-8973-77ae5f2ba0b7","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:24:53.656Z","logger":"controllers.NetworkPolicy","msg":"Reconciling NetworkPolicies for RayCluster","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"d9570756-a210-4350-b23d-b8b630503289","cluster":"stop-lwhxj","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:24:53.656Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"7a1a8827-c2d4-473c-8973-77ae5f2ba0b7"}
{"level":"info","ts":"2026-11-03T15:24:53.656Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"7a1a8827-c2d4-473c-8973-77ae5f2ba0b7","RayCluster":"stop-lwhxj","State":""}
{"level":"info","ts":"2026-11-03T15:24:53.656Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"7a1a8827-c2d4-473c-8973-77ae5f2ba0b7","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:24:53.656Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"stop-lwhxj","reconcileID":"5b6f01a4-b2f5-41a3-a3d6-1ea33b1c3353","namespacedName":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:24:53.656Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"b6f07ae0-4b4e-44c2-a188-7674e4fb874d"}
{"level":"info","ts":"2026-11-03T15:24:53.656Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"b6f07ae0-4b4e-44c2-a188-7674e4fb874d"}
{"level":"info","ts":"2026-11-03T15:24:53.656Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"stop-lwhxj","reconcileID":"5b6f01a4-b2f5-41a3-a3d6-1ea33b1c3353","mode":"IntegratedOAuth","cluster":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:24:53.656Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"stop-lwhxj","reconcileID":"5b6f01a4-b2f5-41a3-a3d6-1ea33b1c3353","cluster":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:24:53.656Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"stop-lwhxj","reconcileID":"5b6f01a4-b2f5-41a3-a3d6-1ea33b1c3353","cluster":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:24:53.711Z","logger":"controllers.NetworkPolicy","msg":"Successfully created NetworkPolicy","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"d9570756-a210-4350-b23d-b8b630503289","name":"stop-lwhxj-head"}
{"level":"info","ts":"2026-11-03T15:24:53.711Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"b6f07ae0-4b4e-44c2-a188-7674e4fb874d","name":"stop-lwhxj-head-svc"}
{"level":"info","ts":"2026-11-03T15:24:53.711Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"b6f07ae0-4b4e-44c2-a188-7674e4fb874d"}
{"level":"info","ts":"2026-11-03T15:24:53.716Z","logger":"raycluster-resource","msg":"default","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:24:53.716Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"stop-lwhxj","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:24:53.761Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"b6f07ae0-4b4e-44c2-a188-7674e4fb874d","error":"certificate ray-head-cert-stop-lwhxj not found: certificates.cert-manager.io \"ray-head-cert-stop-lwhxj\" not found"}
{"level":"error","ts":"2026-11-03T15:24:53.761Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"b6f07ae0-4b4e-44c2-a188-7674e4fb874d","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-stop-lwhxj not found: certificates.cert-manager.io \"ray-head-cert-stop-lwhxj\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:24:53.761Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"b6f07ae0-4b4e-44c2-a188-7674e4fb874d","error":"mTLS secrets not ready: certificate ray-head-cert-stop-lwhxj not found: certificates.cert-manager.io \"ray-head-cert-stop-lwhxj\" not found"}
{"level":"info","ts":"2026-11-03T15:24:53.761Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"b6f07ae0-4b4e-44c2-a188-7674e4fb874d","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"stop-lwhxj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:24:53.761Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"b6f07ae0-4b4e-44c2-a188-7674e4fb874d","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"stop-lwhxj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:24:53.761Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"b6f07ae0-4b4e-44c2-a188-7674e4fb874d","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"stop-lwhxj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:24:53.761Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"b6f07ae0-4b4e-44c2-a188-7674e4fb874d","name":"stop-lwhxj","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:24:53.764Z","logger":"controllers.NetworkPolicy","msg":"Successfully created NetworkPolicy","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"d9570756-a210-4350-b23d-b8b630503289","name":"stop-lwhxj-workers-small-group"}
{"level":"info","ts":"2026-11-03T15:24:53.764Z","logger":"controllers.NetworkPolicy","msg":"Successfully reconciled NetworkPolicies for RayCluster","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"d9570756-a210-4350-b23d-b8b630503289","cluster":"stop-lwhxj","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:24:53.764Z","logger":"controllers.NetworkPolicy","msg":"Reconciling NetworkPolicies for RayCluster","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"a0a3b296-4041-4eb0-947e-e24ec7a3e0bf","cluster":"stop-lwhxj","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:24:53.769Z","logger":"raycluster-resource","msg":"validate update","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:24:53.822Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"b6f07ae0-4b4e-44c2-a188-7674e4fb874d"}
{"level":"error","ts":"2026-11-03T15:24:53.822Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"b6f07ae0-4b4e-44c2-a188-7674e4fb874d","error":"mTLS secrets not ready: certificate ray-head-cert-stop-lwhxj not found: certificates.cert-manager.io \"ray-head-cert-stop-lwhxj\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:24:53.822Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"416a960c-55f6-413b-953f-4c62b5f725f3"}
{"level":"info","ts":"2026-11-03T15:24:53.822Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"416a960c-55f6-413b-953f-4c62b5f725f3"}
{"level":"info","ts":"2026-11-03T15:24:53.822Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"416a960c-55f6-413b-953f-4c62b5f725f3","1 head service found":"stop-lwhxj-head-svc"}
{"level":"info","ts":"2026-11-03T15:24:53.822Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"416a960c-55f6-413b-953f-4c62b5f725f3"}
{"level":"info","ts":"2026-11-03T15:24:53.823Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"d2b58a41-0b13-4afb-a05b-99aa926122ce","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:24:53.823Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"d2b58a41-0b13-4afb-a05b-99aa926122ce","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:24:53.823Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"d2b58a41-0b13-4afb-a05b-99aa926122ce","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:24:53.823Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"d2b58a41-0b13-4afb-a05b-99aa926122ce"}
{"level":"info","ts":"2026-11-03T15:24:53.823Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"d2b58a41-0b13-4afb-a05b-99aa926122ce","RayCluster":"stop-lwhxj","State":""}
{"level":"info","ts":"2026-11-03T15:24:53.823Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"d2b58a41-0b13-4afb-a05b-99aa926122ce","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:24:53.823Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"d2b58a41-0b13-4afb-a05b-99aa926122ce","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:24:53.859Z","logger":"raycluster-resource","msg":"default","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:24:53.859Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"stop-lwhxj","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:24:53.874Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"416a960c-55f6-413b-953f-4c62b5f725f3","error":"certificate ray-head-cert-stop-lwhxj not found: certificates.cert-manager.io \"ray-head-cert-stop-lwhxj\" not found"}
{"level":"error","ts":"2026-11-03T15:24:53.874Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"416a960c-55f6-413b-953f-4c62b5f725f3","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-stop-lwhxj not found: certificates.cert-manager.io \"ray-head-cert-stop-lwhxj\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:24:53.874Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"416a960c-55f6-413b-953f-4c62b5f725f3","error":"mTLS secrets not ready: certificate ray-head-cert-stop-lwhxj not found: certificates.cert-manager.io \"ray-head-cert-stop-lwhxj\" not found"}
{"level":"info","ts":"2026-11-03T15:24:53.874Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"416a960c-55f6-413b-953f-4c62b5f725f3","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"stop-lwhxj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:24:53.874Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"416a960c-55f6-413b-953f-4c62b5f725f3","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"stop-lwhxj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:24:53.874Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"416a960c-55f6-413b-953f-4c62b5f725f3","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"stop-lwhxj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:24:53.874Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"416a960c-55f6-413b-953f-4c62b5f725f3","name":"stop-lwhxj","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:24:53.876Z","logger":"controllers.NetworkPolicy","msg":"Successfully reconciled NetworkPolicies for RayCluster","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"a0a3b296-4041-4eb0-947e-e24ec7a3e0bf","cluster":"stop-lwhxj","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:24:53.883Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"e5ffc7cc-6082-4bd9-9a4e-40f79e02af2d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:24:53.883Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"e5ffc7cc-6082-4bd9-9a4e-40f79e02af2d","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:24:53.883Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"e5ffc7cc-6082-4bd9-9a4e-40f79e02af2d","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:24:53.883Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"e5ffc7cc-6082-4bd9-9a4e-40f79e02af2d"}
{"level":"info","ts":"2026-11-03T15:24:53.883Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"e5ffc7cc-6082-4bd9-9a4e-40f79e02af2d","RayCluster":"stop-lwhxj","State":""}
{"level":"info","ts":"2026-11-03T15:24:53.883Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"e5ffc7cc-6082-4bd9-9a4e-40f79e02af2d","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"error","ts":"2026-11-03T15:24:53.908Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"stop-lwhxj","reconcileID":"5b6f01a4-b2f5-41a3-a3d6-1ea33b1c3353","error":"failed to add finalizer: Operation cannot be fulfilled on rayclusters.ray.io \"stop-lwhxj\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:167\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:24:53.908Z","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"stop-lwhxj","reconcileID":"5b6f01a4-b2f5-41a3-a3d6-1ea33b1c3353"}
{"level":"error","ts":"2026-11-03T15:24:53.908Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"stop-lwhxj","reconcileID":"5b6f01a4-b2f5-41a3-a3d6-1ea33b1c3353","error":"failed to add finalizer: Operation cannot be fulfilled on rayclusters.ray.io \"stop-lwhxj\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:24:53.908Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"stop-lwhxj","reconcileID":"f67f4410-7b12-4d90-9ca8-d3ce7f632640","namespacedName":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:24:53.908Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"stop-lwhxj","reconcileID":"f67f4410-7b12-4d90-9ca8-d3ce7f632640","mode":"IntegratedOAuth","cluster":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:24:53.908Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"stop-lwhxj","reconcileID":"f67f4410-7b12-4d90-9ca8-d3ce7f632640","cluster":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:24:53.908Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"stop-lwhxj","reconcileID":"f67f4410-7b12-4d90-9ca8-d3ce7f632640","cluster":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:24:53.929Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"416a960c-55f6-413b-953f-4c62b5f725f3","name":"stop-lwhxj","error":"Operation cannot be fulfilled on rayclusters.ray.io \"stop-lwhxj\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-kbx4w","name":"stop-lwhxj"}}
{"level":"info","ts":"2026-11-03T15:24:53.929Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"416a960c-55f6-413b-953f-4c62b5f725f3"}
{"level":"error","ts":"2026-11-03T15:24:53.929Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"416a960c-55f6-413b-953f-4c62b5f725f3","error":"mTLS secrets not ready: certificate ray-head-cert-stop-lwhxj not found: certificates.cert-manager.io \"ray-head-cert-stop-lwhxj\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:24:53.939Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"81475e4f-48d2-4fd0-ad1c-d74e25782152"}
{"level":"info","ts":"2026-11-03T15:24:53.939Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"81475e4f-48d2-4fd0-ad1c-d74e25782152"}
{"level":"info","ts":"2026-11-03T15:24:53.939Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"81475e4f-48d2-4fd0-ad1c-d74e25782152","1 head service found":"stop-lwhxj-head-svc"}
{"level":"info","ts":"2026-11-03T15:24:53.939Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"81475e4f-48d2-4fd0-ad1c-d74e25782152"}
{"level":"info","ts":"2026-11-03T15:24:53.966Z","logger":"raycluster-resource","msg":"default","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:24:53.966Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"stop-lwhxj","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:24:53.989Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"81475e4f-48d2-4fd0-ad1c-d74e25782152","error":"certificate ray-head-cert-stop-lwhxj not found: certificates.cert-manager.io \"ray-head-cert-stop-lwhxj\" not found"}
{"level":"error","ts":"2026-11-03T15:24:53.989Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"81475e4f-48d2-4fd0-ad1c-d74e25782152","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-stop-lwhxj not found: certificates.cert-manager.io \"ray-head-cert-stop-lwhxj\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:24:53.990Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"81475e4f-48d2-4fd0-ad1c-d74e25782152","error":"mTLS secrets not ready: certificate ray-head-cert-stop-lwhxj not found: certificates.cert-manager.io \"ray-head-cert-stop-lwhxj\" not found"}
{"level":"info","ts":"2026-11-03T15:24:53.990Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"81475e4f-48d2-4fd0-ad1c-d74e25782152","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"stop-lwhxj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:24:53.990Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"81475e4f-48d2-4fd0-ad1c-d74e25782152","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"stop-lwhxj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:24:53.990Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"81475e4f-48d2-4fd0-ad1c-d74e25782152","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"stop-lwhxj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:24:53.990Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"81475e4f-48d2-4fd0-ad1c-d74e25782152"}
{"level":"error","ts":"2026-11-03T15:24:53.990Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"81475e4f-48d2-4fd0-ad1c-d74e25782152","error":"mTLS secrets not ready: certificate ray-head-cert-stop-lwhxj not found: certificates.cert-manager.io \"ray-head-cert-stop-lwhxj\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:24:54.010Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"dc17353c-befc-428d-99da-f286fb3039de"}
{"level":"info","ts":"2026-11-03T15:24:54.010Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"dc17353c-befc-428d-99da-f286fb3039de"}
{"level":"info","ts":"2026-11-03T15:24:54.010Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"dc17353c-befc-428d-99da-f286fb3039de","1 head service found":"stop-lwhxj-head-svc"}
{"level":"info","ts":"2026-11-03T15:24:54.010Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"dc17353c-befc-428d-99da-f286fb3039de"}
{"level":"info","ts":"2026-11-03T15:24:54.019Z","logger":"raycluster-resource","msg":"validate update","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:24:54.060Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"dc17353c-befc-428d-99da-f286fb3039de","error":"certificate ray-head-cert-stop-lwhxj not found: certificates.cert-manager.io \"ray-head-cert-stop-lwhxj\" not found"}
{"level":"error","ts":"2026-11-03T15:24:54.060Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"dc17353c-befc-428d-99da-f286fb3039de","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-stop-lwhxj not found: certificates.cert-manager.io \"ray-head-cert-stop-lwhxj\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:24:54.060Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"dc17353c-befc-428d-99da-f286fb3039de","error":"mTLS secrets not ready: certificate ray-head-cert-stop-lwhxj not found: certificates.cert-manager.io \"ray-head-cert-stop-lwhxj\" not found"}
{"level":"info","ts":"2026-11-03T15:24:54.060Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"dc17353c-befc-428d-99da-f286fb3039de","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"stop-lwhxj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:24:54.060Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"dc17353c-befc-428d-99da-f286fb3039de","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"stop-lwhxj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:24:54.060Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"dc17353c-befc-428d-99da-f286fb3039de","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"stop-lwhxj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:24:54.060Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"dc17353c-befc-428d-99da-f286fb3039de"}
{"level":"error","ts":"2026-11-03T15:24:54.060Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"dc17353c-befc-428d-99da-f286fb3039de","error":"mTLS secrets not ready: certificate ray-head-cert-stop-lwhxj not found: certificates.cert-manager.io \"ray-head-cert-stop-lwhxj\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:24:54.074Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"2b83ff66-cf73-4e35-a857-b6cd4d2ffea7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:24:54.074Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"2b83ff66-cf73-4e35-a857-b6cd4d2ffea7","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:24:54.074Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"2b83ff66-cf73-4e35-a857-b6cd4d2ffea7","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:24:54.075Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"2b83ff66-cf73-4e35-a857-b6cd4d2ffea7"}
{"level":"info","ts":"2026-11-03T15:24:54.075Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"2b83ff66-cf73-4e35-a857-b6cd4d2ffea7","RayCluster":"stop-lwhxj","State":""}
{"level":"info","ts":"2026-11-03T15:24:54.075Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"2b83ff66-cf73-4e35-a857-b6cd4d2ffea7","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:24:54.101Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"c443bd71-91cc-4714-a95d-ec3f70ca6785"}
{"level":"info","ts":"2026-11-03T15:24:54.101Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"c443bd71-91cc-4714-a95d-ec3f70ca6785"}
{"level":"info","ts":"2026-11-03T15:24:54.101Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"c443bd71-91cc-4714-a95d-ec3f70ca6785","1 head service found":"stop-lwhxj-head-svc"}
{"level":"info","ts":"2026-11-03T15:24:54.101Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"c443bd71-91cc-4714-a95d-ec3f70ca6785"}
{"level":"info","ts":"2026-11-03T15:24:54.138Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"stop-lwhxj","reconcileID":"f67f4410-7b12-4d90-9ca8-d3ce7f632640","name":"stop-lwhxj-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-11-03T15:24:54.152Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"c443bd71-91cc-4714-a95d-ec3f70ca6785","error":"certificate ray-head-cert-stop-lwhxj not found: certificates.cert-manager.io \"ray-head-cert-stop-lwhxj\" not found"}
{"level":"error","ts":"2026-11-03T15:24:54.152Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"c443bd71-91cc-4714-a95d-ec3f70ca6785","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-stop-lwhxj not found: certificates.cert-manager.io \"ray-head-cert-stop-lwhxj\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:24:54.152Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"c443bd71-91cc-4714-a95d-ec3f70ca6785","error":"mTLS secrets not ready: certificate ray-head-cert-stop-lwhxj not found: certificates.cert-manager.io \"ray-head-cert-stop-lwhxj\" not found"}
{"level":"info","ts":"2026-11-03T15:24:54.152Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"c443bd71-91cc-4714-a95d-ec3f70ca6785","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"stop-lwhxj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:24:54.152Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"c443bd71-91cc-4714-a95d-ec3f70ca6785","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"stop-lwhxj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:24:54.152Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"c443bd71-91cc-4714-a95d-ec3f70ca6785","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"stop-lwhxj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:24:54.152Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"c443bd71-91cc-4714-a95d-ec3f70ca6785"}
{"level":"error","ts":"2026-11-03T15:24:54.152Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"c443bd71-91cc-4714-a95d-ec3f70ca6785","error":"mTLS secrets not ready: certificate ray-head-cert-stop-lwhxj not found: certificates.cert-manager.io \"ray-head-cert-stop-lwhxj\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:24:54.201Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"stop-lwhxj","reconcileID":"f67f4410-7b12-4d90-9ca8-d3ce7f632640","name":"kuberay-gateway-access","namespace":"test-ns-kbx4w","operation":"created"}
{"level":"info","ts":"2026-11-03T15:24:54.232Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"00573a7a-6729-402d-b932-1ae1afe48e6a"}
{"level":"info","ts":"2026-11-03T15:24:54.232Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"00573a7a-6729-402d-b932-1ae1afe48e6a"}
{"level":"info","ts":"2026-11-03T15:24:54.232Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"00573a7a-6729-402d-b932-1ae1afe48e6a","1 head service found":"stop-lwhxj-head-svc"}
{"level":"info","ts":"2026-11-03T15:24:54.232Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"00573a7a-6729-402d-b932-1ae1afe48e6a"}
{"level":"info","ts":"2026-11-03T15:24:54.260Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"stop-lwhxj","reconcileID":"f67f4410-7b12-4d90-9ca8-d3ce7f632640","name":"test-ns-kbx4w-stop-lwhxj","operation":"created"}
{"level":"info","ts":"2026-11-03T15:24:54.282Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"00573a7a-6729-402d-b932-1ae1afe48e6a","error":"certificate ray-head-cert-stop-lwhxj not found: certificates.cert-manager.io \"ray-head-cert-stop-lwhxj\" not found"}
{"level":"error","ts":"2026-11-03T15:24:54.282Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"00573a7a-6729-402d-b932-1ae1afe48e6a","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-stop-lwhxj not found: certificates.cert-manager.io \"ray-head-cert-stop-lwhxj\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:24:54.282Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"00573a7a-6729-402d-b932-1ae1afe48e6a","error":"mTLS secrets not ready: certificate ray-head-cert-stop-lwhxj not found: certificates.cert-manager.io \"ray-head-cert-stop-lwhxj\" not found"}
{"level":"info","ts":"2026-11-03T15:24:54.282Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"00573a7a-6729-402d-b932-1ae1afe48e6a","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"stop-lwhxj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:24:54.282Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"00573a7a-6729-402d-b932-1ae1afe48e6a","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"stop-lwhxj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:24:54.282Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"00573a7a-6729-402d-b932-1ae1afe48e6a","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"stop-lwhxj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:24:54.283Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"00573a7a-6729-402d-b932-1ae1afe48e6a"}
{"level":"error","ts":"2026-11-03T15:24:54.283Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"00573a7a-6729-402d-b932-1ae1afe48e6a","error":"mTLS secrets not ready: certificate ray-head-cert-stop-lwhxj not found: certificates.cert-manager.io \"ray-head-cert-stop-lwhxj\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:24:54.315Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"stop-lwhxj","reconcileID":"f67f4410-7b12-4d90-9ca8-d3ce7f632640","name":"kube-rbac-proxy-config-stop-lwhxj","operation":"created"}
{"level":"error","ts":"2026-11-03T15:24:54.371Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"stop-lwhxj","reconcileID":"f67f4410-7b12-4d90-9ca8-d3ce7f632640","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"stop-lwhxj\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:167\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:24:54.371Z","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"stop-lwhxj","reconcileID":"f67f4410-7b12-4d90-9ca8-d3ce7f632640"}
{"level":"error","ts":"2026-11-03T15:24:54.371Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"stop-lwhxj","reconcileID":"f67f4410-7b12-4d90-9ca8-d3ce7f632640","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"stop-lwhxj\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:24:54.371Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"stop-lwhxj","reconcileID":"94894e1e-2f0a-4366-85c2-679a7a3a34bf","namespacedName":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:24:54.371Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"stop-lwhxj","reconcileID":"94894e1e-2f0a-4366-85c2-679a7a3a34bf","mode":"IntegratedOAuth","cluster":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:24:54.371Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"stop-lwhxj","reconcileID":"94894e1e-2f0a-4366-85c2-679a7a3a34bf","cluster":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:24:54.436Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"stop-lwhxj","reconcileID":"94894e1e-2f0a-4366-85c2-679a7a3a34bf","cluster":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:24:54.436Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"e2f75a4a-134c-4430-a865-c4b0b27c3da7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:24:54.436Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"e2f75a4a-134c-4430-a865-c4b0b27c3da7","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:24:54.436Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"e2f75a4a-134c-4430-a865-c4b0b27c3da7","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:24:54.436Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"e2f75a4a-134c-4430-a865-c4b0b27c3da7"}
{"level":"info","ts":"2026-11-03T15:24:54.436Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"e2f75a4a-134c-4430-a865-c4b0b27c3da7","RayCluster":"stop-lwhxj","State":""}
{"level":"info","ts":"2026-11-03T15:24:54.436Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"e2f75a4a-134c-4430-a865-c4b0b27c3da7","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:24:54.436Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"e2f75a4a-134c-4430-a865-c4b0b27c3da7","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:24:54.443Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"edc28f5b-fab7-4d77-b65d-d4beb1afa139"}
{"level":"info","ts":"2026-11-03T15:24:54.443Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"edc28f5b-fab7-4d77-b65d-d4beb1afa139"}
{"level":"info","ts":"2026-11-03T15:24:54.443Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"edc28f5b-fab7-4d77-b65d-d4beb1afa139","1 head service found":"stop-lwhxj-head-svc"}
{"level":"info","ts":"2026-11-03T15:24:54.443Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"edc28f5b-fab7-4d77-b65d-d4beb1afa139"}
{"level":"info","ts":"2026-11-03T15:24:54.496Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"edc28f5b-fab7-4d77-b65d-d4beb1afa139","error":"certificate ray-head-cert-stop-lwhxj is not ready at current generation 1"}
{"level":"error","ts":"2026-11-03T15:24:54.496Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"edc28f5b-fab7-4d77-b65d-d4beb1afa139","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-stop-lwhxj is not ready at current generation 1","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:24:54.496Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"edc28f5b-fab7-4d77-b65d-d4beb1afa139","error":"mTLS secrets not ready: certificate ray-head-cert-stop-lwhxj is not ready at current generation 1"}
{"level":"info","ts":"2026-11-03T15:24:54.496Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"edc28f5b-fab7-4d77-b65d-d4beb1afa139","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"stop-lwhxj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:24:54.496Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"edc28f5b-fab7-4d77-b65d-d4beb1afa139","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"stop-lwhxj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:24:54.496Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"edc28f5b-fab7-4d77-b65d-d4beb1afa139","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"stop-lwhxj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:24:54.496Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"edc28f5b-fab7-4d77-b65d-d4beb1afa139"}
{"level":"error","ts":"2026-11-03T15:24:54.496Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"edc28f5b-fab7-4d77-b65d-d4beb1afa139","error":"mTLS secrets not ready: certificate ray-head-cert-stop-lwhxj is not ready at current generation 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:24:54.498Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"5a74e976-eed5-4c91-ba10-5b0af70b1d7c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:24:54.498Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"5a74e976-eed5-4c91-ba10-5b0af70b1d7c","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:24:54.498Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"5a74e976-eed5-4c91-ba10-5b0af70b1d7c","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:24:54.498Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"5a74e976-eed5-4c91-ba10-5b0af70b1d7c"}
{"level":"info","ts":"2026-11-03T15:24:54.498Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"5a74e976-eed5-4c91-ba10-5b0af70b1d7c","RayCluster":"stop-lwhxj","State":""}
{"level":"info","ts":"2026-11-03T15:24:54.498Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"5a74e976-eed5-4c91-ba10-5b0af70b1d7c","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:24:54.632Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"e4722055-d5fb-41c4-a662-aea8e2a8593d","certificate":"ray-head-cert-stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:24:54.632Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"e4722055-d5fb-41c4-a662-aea8e2a8593d"}
{"level":"info","ts":"2026-11-03T15:24:54.632Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"184602ba-bb95-4223-aed0-4c640f596a0b","cluster":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:24:54.817Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"9793e61b-8753-45bf-b940-1a2500217201"}
{"level":"info","ts":"2026-11-03T15:24:54.817Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"9793e61b-8753-45bf-b940-1a2500217201"}
{"level":"info","ts":"2026-11-03T15:24:54.817Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"9793e61b-8753-45bf-b940-1a2500217201","1 head service found":"stop-lwhxj-head-svc"}
{"level":"info","ts":"2026-11-03T15:24:54.817Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"9793e61b-8753-45bf-b940-1a2500217201"}
{"level":"info","ts":"2026-11-03T15:24:54.868Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"9793e61b-8753-45bf-b940-1a2500217201","error":"certificate ray-head-cert-stop-lwhxj is not ready at current generation 1"}
{"level":"error","ts":"2026-11-03T15:24:54.868Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"9793e61b-8753-45bf-b940-1a2500217201","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-stop-lwhxj is not ready at current generation 1","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:24:54.868Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"9793e61b-8753-45bf-b940-1a2500217201","error":"mTLS secrets not ready: certificate ray-head-cert-stop-lwhxj is not ready at current generation 1"}
{"level":"info","ts":"2026-11-03T15:24:54.868Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"9793e61b-8753-45bf-b940-1a2500217201","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"stop-lwhxj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:24:54.868Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"9793e61b-8753-45bf-b940-1a2500217201","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"stop-lwhxj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:24:54.868Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"9793e61b-8753-45bf-b940-1a2500217201","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"stop-lwhxj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:24:54.868Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"9793e61b-8753-45bf-b940-1a2500217201"}
{"level":"error","ts":"2026-11-03T15:24:54.868Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"9793e61b-8753-45bf-b940-1a2500217201","error":"mTLS secrets not ready: certificate ray-head-cert-stop-lwhxj is not ready at current generation 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:24:55.090Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"184602ba-bb95-4223-aed0-4c640f596a0b","certificate":"ray-head-cert-stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:24:55.090Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"184602ba-bb95-4223-aed0-4c640f596a0b"}
{"level":"info","ts":"2026-11-03T15:24:55.508Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"7c886b21-6ade-46c9-a9f4-1315e1e4c78b"}
{"level":"info","ts":"2026-11-03T15:24:55.508Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"7c886b21-6ade-46c9-a9f4-1315e1e4c78b"}
{"level":"info","ts":"2026-11-03T15:24:55.508Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"7c886b21-6ade-46c9-a9f4-1315e1e4c78b","1 head service found":"stop-lwhxj-head-svc"}
{"level":"info","ts":"2026-11-03T15:24:55.508Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"7c886b21-6ade-46c9-a9f4-1315e1e4c78b"}
{"level":"info","ts":"2026-11-03T15:24:55.559Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"7c886b21-6ade-46c9-a9f4-1315e1e4c78b","error":"certificate ray-head-cert-stop-lwhxj is not ready at current generation 1"}
{"level":"error","ts":"2026-11-03T15:24:55.559Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"7c886b21-6ade-46c9-a9f4-1315e1e4c78b","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-stop-lwhxj is not ready at current generation 1","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:24:55.559Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"7c886b21-6ade-46c9-a9f4-1315e1e4c78b","error":"mTLS secrets not ready: certificate ray-head-cert-stop-lwhxj is not ready at current generation 1"}
{"level":"info","ts":"2026-11-03T15:24:55.559Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"7c886b21-6ade-46c9-a9f4-1315e1e4c78b","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"stop-lwhxj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:24:55.559Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"7c886b21-6ade-46c9-a9f4-1315e1e4c78b","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"stop-lwhxj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:24:55.559Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"7c886b21-6ade-46c9-a9f4-1315e1e4c78b","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"stop-lwhxj","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:24:55.559Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"7c886b21-6ade-46c9-a9f4-1315e1e4c78b"}
{"level":"error","ts":"2026-11-03T15:24:55.559Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"7c886b21-6ade-46c9-a9f4-1315e1e4c78b","error":"mTLS secrets not ready: certificate ray-head-cert-stop-lwhxj is not ready at current generation 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:24:56.841Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"e00874bd-3e2f-4233-81b3-c4f81367086b"}
{"level":"info","ts":"2026-11-03T15:24:56.841Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"e00874bd-3e2f-4233-81b3-c4f81367086b"}
{"level":"info","ts":"2026-11-03T15:24:56.841Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"e00874bd-3e2f-4233-81b3-c4f81367086b","1 head service found":"stop-lwhxj-head-svc"}
{"level":"info","ts":"2026-11-03T15:24:56.841Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"e00874bd-3e2f-4233-81b3-c4f81367086b"}
{"level":"info","ts":"2026-11-03T15:24:56.943Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"e00874bd-3e2f-4233-81b3-c4f81367086b","mode":"IntegratedOAuth","cluster":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:24:56.943Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"e00874bd-3e2f-4233-81b3-c4f81367086b","cluster":"stop-lwhxj","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-11-03T15:24:56.943Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"e00874bd-3e2f-4233-81b3-c4f81367086b","cluster":"stop-lwhxj","authType":"IntegratedOAuth","serviceAccount":"stop-lwhxj-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-11-03T15:24:56.943Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"e00874bd-3e2f-4233-81b3-c4f81367086b","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"stop-lwhxj","ray.io/group":"headgroup","ray.io/identifier":"stop-lwhxj-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-11-03T15:24:56.943Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"e00874bd-3e2f-4233-81b3-c4f81367086b","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-11-03T15:24:56.943Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"e00874bd-3e2f-4233-81b3-c4f81367086b","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-11-03T15:24:56.943Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"e00874bd-3e2f-4233-81b3-c4f81367086b","rayNodeType":"head","generatedCmd":"ulimit -n ${RAY_START_ULIMIT_OPEN_FILES:-65536}; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-11-03T15:24:56.943Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"e00874bd-3e2f-4233-81b3-c4f81367086b","enabled":true}
{"level":"info","ts":"2026-11-03T15:24:57.053Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"e00874bd-3e2f-4233-81b3-c4f81367086b","name":"stop-lwhxj-head-brfn5"}
{"level":"info","ts":"2026-11-03T15:24:57.053Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"e00874bd-3e2f-4233-81b3-c4f81367086b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:24:57.053Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"e00874bd-3e2f-4233-81b3-c4f81367086b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:24:57.053Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"e00874bd-3e2f-4233-81b3-c4f81367086b","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-11-03T15:24:57.053Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"e00874bd-3e2f-4233-81b3-c4f81367086b","group":"small-group","indices":{}}
{"level":"info","ts":"2026-11-03T15:24:57.053Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"e00874bd-3e2f-4233-81b3-c4f81367086b","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-11-03T15:24:57.159Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"e00874bd-3e2f-4233-81b3-c4f81367086b","creating worker for group":"small-group","index":0,"total":1,"replicaIndex":0}
{"level":"info","ts":"2026-11-03T15:24:57.159Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"e00874bd-3e2f-4233-81b3-c4f81367086b","nodeType":"worker","rayStartParams":{"address":"stop-lwhxj-head-svc.test-ns-kbx4w.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-11-03T15:24:57.159Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"e00874bd-3e2f-4233-81b3-c4f81367086b","rayStartCmd":"ray start  --address=stop-lwhxj-head-svc.test-ns-kbx4w.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-11-03T15:24:57.159Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"e00874bd-3e2f-4233-81b3-c4f81367086b","rayNodeType":"worker","generatedCmd":"ulimit -n ${RAY_START_ULIMIT_OPEN_FILES:-65536}; ray start  --address=stop-lwhxj-head-svc.test-ns-kbx4w.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-11-03T15:24:57.159Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"e00874bd-3e2f-4233-81b3-c4f81367086b","enabled":true}
{"level":"info","ts":"2026-11-03T15:24:57.229Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"e00874bd-3e2f-4233-81b3-c4f81367086b","name":"stop-lwhxj-small-group-worker-zxh4p"}
{"level":"info","ts":"2026-11-03T15:24:57.229Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"e00874bd-3e2f-4233-81b3-c4f81367086b","podName":"stop-lwhxj-small-group-worker-zxh4p","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:24:57.229Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"e00874bd-3e2f-4233-81b3-c4f81367086b","podName":"stop-lwhxj-small-group-worker-zxh4p","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:24:57.229Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"e00874bd-3e2f-4233-81b3-c4f81367086b","name":"stop-lwhxj","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:57Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:24:57.292Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"d218991e-0b5e-420b-afd6-5fa1df206ffe"}
{"level":"info","ts":"2026-11-03T15:24:57.292Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"d218991e-0b5e-420b-afd6-5fa1df206ffe"}
{"level":"info","ts":"2026-11-03T15:24:57.292Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"d218991e-0b5e-420b-afd6-5fa1df206ffe","1 head service found":"stop-lwhxj-head-svc"}
{"level":"info","ts":"2026-11-03T15:24:57.292Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"32ce2da3-837e-4e92-8278-3382f1bc5987","cluster":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:24:57.292Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"d218991e-0b5e-420b-afd6-5fa1df206ffe","Found 1 head Pod":"stop-lwhxj-head-brfn5","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:24:57.292Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"d218991e-0b5e-420b-afd6-5fa1df206ffe","head Pod":"stop-lwhxj-head-brfn5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-lwhxj-head-brfn5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:24:57.292Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"d218991e-0b5e-420b-afd6-5fa1df206ffe","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:24:57.292Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"d218991e-0b5e-420b-afd6-5fa1df206ffe","worker Pod":"stop-lwhxj-small-group-worker-zxh4p","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-lwhxj-small-group-worker-zxh4p. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:24:57.292Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"d218991e-0b5e-420b-afd6-5fa1df206ffe","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:24:57.292Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"d218991e-0b5e-420b-afd6-5fa1df206ffe","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:24:57.292Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"d218991e-0b5e-420b-afd6-5fa1df206ffe","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:24:57.292Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"d218991e-0b5e-420b-afd6-5fa1df206ffe","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:24:57.292Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"d218991e-0b5e-420b-afd6-5fa1df206ffe","podName":"stop-lwhxj-small-group-worker-zxh4p","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:24:57.292Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"96c66381-45bf-4158-a52c-8f93a0f2620a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:24:57.292Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"96c66381-45bf-4158-a52c-8f93a0f2620a","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:24:57.292Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"d218991e-0b5e-420b-afd6-5fa1df206ffe","podName":"stop-lwhxj-small-group-worker-zxh4p","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:24:57.292Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"96c66381-45bf-4158-a52c-8f93a0f2620a","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:24:57.292Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"d218991e-0b5e-420b-afd6-5fa1df206ffe","name":"stop-lwhxj","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:57Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:24:57.293Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"96c66381-45bf-4158-a52c-8f93a0f2620a"}
{"level":"info","ts":"2026-11-03T15:24:57.293Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"96c66381-45bf-4158-a52c-8f93a0f2620a","RayCluster":"stop-lwhxj","State":""}
{"level":"info","ts":"2026-11-03T15:24:57.293Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"96c66381-45bf-4158-a52c-8f93a0f2620a","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:57Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:24:57.293Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"96c66381-45bf-4158-a52c-8f93a0f2620a","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:24:57.350Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"d218991e-0b5e-420b-afd6-5fa1df206ffe","name":"stop-lwhxj","error":"Operation cannot be fulfilled on rayclusters.ray.io \"stop-lwhxj\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-kbx4w","name":"stop-lwhxj"}}
{"level":"info","ts":"2026-11-03T15:24:57.350Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"d218991e-0b5e-420b-afd6-5fa1df206ffe"}
{"level":"error","ts":"2026-11-03T15:24:57.350Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"d218991e-0b5e-420b-afd6-5fa1df206ffe","error":"Operation cannot be fulfilled on rayclusters.ray.io \"stop-lwhxj\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:24:57.350Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"ba8e8b52-cbff-4f2b-9db9-5864afaa678e"}
{"level":"info","ts":"2026-11-03T15:24:57.350Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"ba8e8b52-cbff-4f2b-9db9-5864afaa678e"}
{"level":"info","ts":"2026-11-03T15:24:57.350Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"ba8e8b52-cbff-4f2b-9db9-5864afaa678e","1 head service found":"stop-lwhxj-head-svc"}
{"level":"info","ts":"2026-11-03T15:24:57.350Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"ba8e8b52-cbff-4f2b-9db9-5864afaa678e","Found 1 head Pod":"stop-lwhxj-head-brfn5","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:24:57.350Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"ba8e8b52-cbff-4f2b-9db9-5864afaa678e","head Pod":"stop-lwhxj-head-brfn5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-lwhxj-head-brfn5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:24:57.350Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"ba8e8b52-cbff-4f2b-9db9-5864afaa678e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:24:57.350Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"ba8e8b52-cbff-4f2b-9db9-5864afaa678e","worker Pod":"stop-lwhxj-small-group-worker-zxh4p","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-lwhxj-small-group-worker-zxh4p. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:24:57.350Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"ba8e8b52-cbff-4f2b-9db9-5864afaa678e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:24:57.350Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"ba8e8b52-cbff-4f2b-9db9-5864afaa678e","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:24:57.350Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"ba8e8b52-cbff-4f2b-9db9-5864afaa678e","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:24:57.350Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"ba8e8b52-cbff-4f2b-9db9-5864afaa678e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:24:57.350Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"ba8e8b52-cbff-4f2b-9db9-5864afaa678e","podName":"stop-lwhxj-head-brfn5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:24:57.350Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"ba8e8b52-cbff-4f2b-9db9-5864afaa678e","podName":"stop-lwhxj-head-brfn5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:24:57.350Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"ba8e8b52-cbff-4f2b-9db9-5864afaa678e","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:24:57.350Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"ba8e8b52-cbff-4f2b-9db9-5864afaa678e","seconds":300}
{"level":"info","ts":"2026-11-03T15:24:57.354Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"2dd83369-6b77-47ef-b64a-3c759b364ab5","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:24:57.354Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"2dd83369-6b77-47ef-b64a-3c759b364ab5","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:24:57.354Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"2dd83369-6b77-47ef-b64a-3c759b364ab5","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:24:57.354Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"2dd83369-6b77-47ef-b64a-3c759b364ab5"}
{"level":"info","ts":"2026-11-03T15:24:57.354Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"2dd83369-6b77-47ef-b64a-3c759b364ab5","RayCluster":"stop-lwhxj","State":""}
{"level":"info","ts":"2026-11-03T15:24:57.354Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"2dd83369-6b77-47ef-b64a-3c759b364ab5","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:57Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:57Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:24:57.368Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"d5b32607-6a29-45e7-8779-c5a955c36ecf"}
{"level":"info","ts":"2026-11-03T15:24:57.368Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"d5b32607-6a29-45e7-8779-c5a955c36ecf"}
{"level":"info","ts":"2026-11-03T15:24:57.368Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"d5b32607-6a29-45e7-8779-c5a955c36ecf","1 head service found":"stop-lwhxj-head-svc"}
{"level":"info","ts":"2026-11-03T15:24:57.368Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"d5b32607-6a29-45e7-8779-c5a955c36ecf","Found 1 head Pod":"stop-lwhxj-head-brfn5","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:24:57.368Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"d5b32607-6a29-45e7-8779-c5a955c36ecf","head Pod":"stop-lwhxj-head-brfn5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-lwhxj-head-brfn5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:24:57.368Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"d5b32607-6a29-45e7-8779-c5a955c36ecf","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:24:57.368Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"d5b32607-6a29-45e7-8779-c5a955c36ecf","worker Pod":"stop-lwhxj-small-group-worker-zxh4p","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-lwhxj-small-group-worker-zxh4p. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:24:57.368Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"d5b32607-6a29-45e7-8779-c5a955c36ecf","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:24:57.368Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"d5b32607-6a29-45e7-8779-c5a955c36ecf","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:24:57.368Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"d5b32607-6a29-45e7-8779-c5a955c36ecf","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:24:57.368Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"d5b32607-6a29-45e7-8779-c5a955c36ecf","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:24:57.368Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"d5b32607-6a29-45e7-8779-c5a955c36ecf","podName":"stop-lwhxj-head-brfn5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:24:57.368Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"d5b32607-6a29-45e7-8779-c5a955c36ecf","podName":"stop-lwhxj-head-brfn5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:24:57.369Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"d5b32607-6a29-45e7-8779-c5a955c36ecf","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:24:57.369Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"d5b32607-6a29-45e7-8779-c5a955c36ecf","seconds":300}
{"level":"info","ts":"2026-11-03T15:24:57.986Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"170dc44c-53e2-4885-9683-142297aefa2f"}
{"level":"info","ts":"2026-11-03T15:24:57.986Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"170dc44c-53e2-4885-9683-142297aefa2f"}
{"level":"info","ts":"2026-11-03T15:24:57.986Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"170dc44c-53e2-4885-9683-142297aefa2f","1 head service found":"stop-lwhxj-head-svc"}
{"level":"info","ts":"2026-11-03T15:24:57.986Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"170dc44c-53e2-4885-9683-142297aefa2f","Found 1 head Pod":"stop-lwhxj-head-brfn5","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:24:57.986Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"170dc44c-53e2-4885-9683-142297aefa2f","head Pod":"stop-lwhxj-head-brfn5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-lwhxj-head-brfn5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:24:57.986Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"170dc44c-53e2-4885-9683-142297aefa2f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:24:57.986Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"170dc44c-53e2-4885-9683-142297aefa2f","worker Pod":"stop-lwhxj-small-group-worker-zxh4p","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-lwhxj-small-group-worker-zxh4p. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:24:57.986Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"170dc44c-53e2-4885-9683-142297aefa2f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:24:57.986Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"170dc44c-53e2-4885-9683-142297aefa2f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:24:57.986Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"170dc44c-53e2-4885-9683-142297aefa2f","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:24:57.986Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"170dc44c-53e2-4885-9683-142297aefa2f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:24:57.986Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"170dc44c-53e2-4885-9683-142297aefa2f","podName":"stop-lwhxj-head-brfn5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:24:57.986Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"170dc44c-53e2-4885-9683-142297aefa2f","podName":"stop-lwhxj-head-brfn5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:24:57.986Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"170dc44c-53e2-4885-9683-142297aefa2f","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:24:57.986Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"170dc44c-53e2-4885-9683-142297aefa2f","seconds":300}
{"level":"info","ts":"2026-11-03T15:24:58.113Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"86ce6135-154f-4a6f-9a3b-04a45d5afb9c"}
{"level":"info","ts":"2026-11-03T15:24:58.113Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"86ce6135-154f-4a6f-9a3b-04a45d5afb9c"}
{"level":"info","ts":"2026-11-03T15:24:58.113Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"86ce6135-154f-4a6f-9a3b-04a45d5afb9c","1 head service found":"stop-lwhxj-head-svc"}
{"level":"info","ts":"2026-11-03T15:24:58.114Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"86ce6135-154f-4a6f-9a3b-04a45d5afb9c","Found 1 head Pod":"stop-lwhxj-head-brfn5","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:24:58.114Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"86ce6135-154f-4a6f-9a3b-04a45d5afb9c","head Pod":"stop-lwhxj-head-brfn5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-lwhxj-head-brfn5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:24:58.114Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"86ce6135-154f-4a6f-9a3b-04a45d5afb9c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:24:58.114Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"86ce6135-154f-4a6f-9a3b-04a45d5afb9c","worker Pod":"stop-lwhxj-small-group-worker-zxh4p","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-lwhxj-small-group-worker-zxh4p. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:24:58.114Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"86ce6135-154f-4a6f-9a3b-04a45d5afb9c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:24:58.114Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"86ce6135-154f-4a6f-9a3b-04a45d5afb9c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:24:58.114Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"86ce6135-154f-4a6f-9a3b-04a45d5afb9c","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:24:58.114Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"86ce6135-154f-4a6f-9a3b-04a45d5afb9c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:24:58.114Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"86ce6135-154f-4a6f-9a3b-04a45d5afb9c","podName":"stop-lwhxj-head-brfn5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:24:58.114Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"86ce6135-154f-4a6f-9a3b-04a45d5afb9c","podName":"stop-lwhxj-head-brfn5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:24:58.114Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"86ce6135-154f-4a6f-9a3b-04a45d5afb9c","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:24:58.114Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"86ce6135-154f-4a6f-9a3b-04a45d5afb9c","seconds":300}
{"level":"info","ts":"2026-11-03T15:24:58.299Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"a9e62394-e449-4e83-99db-b19d9581b35c"}
{"level":"info","ts":"2026-11-03T15:24:58.299Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"a9e62394-e449-4e83-99db-b19d9581b35c"}
{"level":"info","ts":"2026-11-03T15:24:58.299Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"a9e62394-e449-4e83-99db-b19d9581b35c","1 head service found":"stop-lwhxj-head-svc"}
{"level":"info","ts":"2026-11-03T15:24:58.299Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"a9e62394-e449-4e83-99db-b19d9581b35c","Found 1 head Pod":"stop-lwhxj-head-brfn5","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:24:58.299Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"a9e62394-e449-4e83-99db-b19d9581b35c","head Pod":"stop-lwhxj-head-brfn5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-lwhxj-head-brfn5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:24:58.299Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"a9e62394-e449-4e83-99db-b19d9581b35c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:24:58.299Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"a9e62394-e449-4e83-99db-b19d9581b35c","worker Pod":"stop-lwhxj-small-group-worker-zxh4p","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-lwhxj-small-group-worker-zxh4p. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:24:58.299Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"a9e62394-e449-4e83-99db-b19d9581b35c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:24:58.299Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"a9e62394-e449-4e83-99db-b19d9581b35c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:24:58.299Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"a9e62394-e449-4e83-99db-b19d9581b35c","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:24:58.299Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"a9e62394-e449-4e83-99db-b19d9581b35c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:24:58.299Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"a9e62394-e449-4e83-99db-b19d9581b35c","podName":"stop-lwhxj-small-group-worker-zxh4p","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:24:58.300Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"a9e62394-e449-4e83-99db-b19d9581b35c","podName":"stop-lwhxj-small-group-worker-zxh4p","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:24:58.300Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"a9e62394-e449-4e83-99db-b19d9581b35c","name":"stop-lwhxj","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:57Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:24:58.361Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"e838557f-1046-409c-a175-6d6bdb9de1e6","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:24:58.361Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"e838557f-1046-409c-a175-6d6bdb9de1e6","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:24:58.361Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"e838557f-1046-409c-a175-6d6bdb9de1e6","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:24:58.361Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"e42999d1-9913-45cd-89de-6e4c523026f2","cluster":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:24:58.361Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"e838557f-1046-409c-a175-6d6bdb9de1e6"}
{"level":"info","ts":"2026-11-03T15:24:58.361Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"e838557f-1046-409c-a175-6d6bdb9de1e6","RayCluster":"stop-lwhxj","State":""}
{"level":"info","ts":"2026-11-03T15:24:58.361Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"e838557f-1046-409c-a175-6d6bdb9de1e6","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:57Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:24:58.361Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"e838557f-1046-409c-a175-6d6bdb9de1e6","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:24:58.366Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"84e6d134-6067-4834-a95b-d9a6989098fc"}
{"level":"info","ts":"2026-11-03T15:24:58.366Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"84e6d134-6067-4834-a95b-d9a6989098fc"}
{"level":"info","ts":"2026-11-03T15:24:58.366Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"84e6d134-6067-4834-a95b-d9a6989098fc","1 head service found":"stop-lwhxj-head-svc"}
{"level":"info","ts":"2026-11-03T15:24:58.366Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"84e6d134-6067-4834-a95b-d9a6989098fc","Found 1 head Pod":"stop-lwhxj-head-brfn5","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:24:58.366Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"84e6d134-6067-4834-a95b-d9a6989098fc","head Pod":"stop-lwhxj-head-brfn5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-lwhxj-head-brfn5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:24:58.366Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"84e6d134-6067-4834-a95b-d9a6989098fc","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:24:58.366Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"84e6d134-6067-4834-a95b-d9a6989098fc","worker Pod":"stop-lwhxj-small-group-worker-zxh4p","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-lwhxj-small-group-worker-zxh4p. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:24:58.366Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"84e6d134-6067-4834-a95b-d9a6989098fc","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:24:58.366Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"84e6d134-6067-4834-a95b-d9a6989098fc","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:24:58.366Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"84e6d134-6067-4834-a95b-d9a6989098fc","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:24:58.366Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"84e6d134-6067-4834-a95b-d9a6989098fc","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:24:58.366Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"84e6d134-6067-4834-a95b-d9a6989098fc","podName":"stop-lwhxj-head-brfn5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:24:58.366Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"84e6d134-6067-4834-a95b-d9a6989098fc","podName":"stop-lwhxj-head-brfn5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:24:58.366Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"84e6d134-6067-4834-a95b-d9a6989098fc","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:24:58.366Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"84e6d134-6067-4834-a95b-d9a6989098fc","seconds":300}
{"level":"info","ts":"2026-11-03T15:24:58.423Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"0a336125-d836-4c4a-98d4-05c4525d7229","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:24:58.423Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"0a336125-d836-4c4a-98d4-05c4525d7229","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:24:58.423Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"0a336125-d836-4c4a-98d4-05c4525d7229","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:24:58.423Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"0a336125-d836-4c4a-98d4-05c4525d7229"}
{"level":"info","ts":"2026-11-03T15:24:58.423Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"0a336125-d836-4c4a-98d4-05c4525d7229","RayCluster":"stop-lwhxj","State":""}
{"level":"info","ts":"2026-11-03T15:24:58.423Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"0a336125-d836-4c4a-98d4-05c4525d7229","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:24:58.666Z","logger":"controllers.RayClusterMTLS","msg":"Updating head certificate","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"e42999d1-9913-45cd-89de-6e4c523026f2","certificate":"ray-head-cert-stop-lwhxj","ipAddresses":["10.132.0.41","127.0.0.1"]}
{"level":"info","ts":"2026-11-03T15:24:58.929Z","logger":"controllers.RayClusterMTLS","msg":"Updating worker certificate","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"e42999d1-9913-45cd-89de-6e4c523026f2","certificate":"ray-worker-cert-stop-lwhxj","ipAddresses":["10.134.0.45","127.0.0.1"]}
{"level":"info","ts":"2026-11-03T15:24:59.191Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"e42999d1-9913-45cd-89de-6e4c523026f2","certificate":"ray-head-cert-stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:24:59.191Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"e42999d1-9913-45cd-89de-6e4c523026f2"}
{"level":"info","ts":"2026-11-03T15:25:01.424Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"3729552d-950e-4557-8f26-aade9267432d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:25:01.424Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"3729552d-950e-4557-8f26-aade9267432d","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:25:01.424Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"3729552d-950e-4557-8f26-aade9267432d","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:25:01.424Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"3729552d-950e-4557-8f26-aade9267432d"}
{"level":"info","ts":"2026-11-03T15:25:01.424Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"3729552d-950e-4557-8f26-aade9267432d","RayCluster":"stop-lwhxj","State":""}
{"level":"info","ts":"2026-11-03T15:25:01.424Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"3729552d-950e-4557-8f26-aade9267432d","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:25:04.425Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"01c3ff6e-5ab5-4ba9-b271-035e6e905a8b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:25:04.425Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"01c3ff6e-5ab5-4ba9-b271-035e6e905a8b","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:25:04.425Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"01c3ff6e-5ab5-4ba9-b271-035e6e905a8b","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:25:04.425Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"01c3ff6e-5ab5-4ba9-b271-035e6e905a8b"}
{"level":"info","ts":"2026-11-03T15:25:04.425Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"01c3ff6e-5ab5-4ba9-b271-035e6e905a8b","RayCluster":"stop-lwhxj","State":""}
{"level":"info","ts":"2026-11-03T15:25:04.425Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"01c3ff6e-5ab5-4ba9-b271-035e6e905a8b","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:25:07.426Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"b29cb4e8-6bfe-4378-90d5-cec8abe3bb1c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:25:07.426Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"b29cb4e8-6bfe-4378-90d5-cec8abe3bb1c","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:25:07.426Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"b29cb4e8-6bfe-4378-90d5-cec8abe3bb1c","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:25:07.426Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"b29cb4e8-6bfe-4378-90d5-cec8abe3bb1c"}
{"level":"info","ts":"2026-11-03T15:25:07.426Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"b29cb4e8-6bfe-4378-90d5-cec8abe3bb1c","RayCluster":"stop-lwhxj","State":""}
{"level":"info","ts":"2026-11-03T15:25:07.426Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"b29cb4e8-6bfe-4378-90d5-cec8abe3bb1c","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:25:10.427Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"8b3efde9-41bf-45c2-a715-93818c4cf5ee","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:25:10.427Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"8b3efde9-41bf-45c2-a715-93818c4cf5ee","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:25:10.427Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"8b3efde9-41bf-45c2-a715-93818c4cf5ee","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:25:10.427Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"8b3efde9-41bf-45c2-a715-93818c4cf5ee"}
{"level":"info","ts":"2026-11-03T15:25:10.427Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"8b3efde9-41bf-45c2-a715-93818c4cf5ee","RayCluster":"stop-lwhxj","State":""}
{"level":"info","ts":"2026-11-03T15:25:10.427Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"8b3efde9-41bf-45c2-a715-93818c4cf5ee","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:25:13.429Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"4b6e9dcd-becc-4273-ae44-4550e6d81033","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:25:13.429Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"4b6e9dcd-becc-4273-ae44-4550e6d81033","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:25:13.429Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"4b6e9dcd-becc-4273-ae44-4550e6d81033","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:25:13.429Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"4b6e9dcd-becc-4273-ae44-4550e6d81033"}
{"level":"info","ts":"2026-11-03T15:25:13.429Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"4b6e9dcd-becc-4273-ae44-4550e6d81033","RayCluster":"stop-lwhxj","State":""}
{"level":"info","ts":"2026-11-03T15:25:13.429Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"4b6e9dcd-becc-4273-ae44-4550e6d81033","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:25:16.430Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"cfc8d9f8-9258-4c3f-a2fb-d67d3edd10f8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:25:16.430Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"cfc8d9f8-9258-4c3f-a2fb-d67d3edd10f8","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:25:16.430Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"cfc8d9f8-9258-4c3f-a2fb-d67d3edd10f8","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:25:16.430Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"cfc8d9f8-9258-4c3f-a2fb-d67d3edd10f8"}
{"level":"info","ts":"2026-11-03T15:25:16.430Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"cfc8d9f8-9258-4c3f-a2fb-d67d3edd10f8","RayCluster":"stop-lwhxj","State":""}
{"level":"info","ts":"2026-11-03T15:25:16.430Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"cfc8d9f8-9258-4c3f-a2fb-d67d3edd10f8","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:25:19.431Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"c74a68e1-46d2-48cb-b5b4-7aa327ce42d0","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:25:19.431Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"c74a68e1-46d2-48cb-b5b4-7aa327ce42d0","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:25:19.431Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"c74a68e1-46d2-48cb-b5b4-7aa327ce42d0","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:25:19.432Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"c74a68e1-46d2-48cb-b5b4-7aa327ce42d0"}
{"level":"info","ts":"2026-11-03T15:25:19.432Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"c74a68e1-46d2-48cb-b5b4-7aa327ce42d0","RayCluster":"stop-lwhxj","State":""}
{"level":"info","ts":"2026-11-03T15:25:19.432Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"c74a68e1-46d2-48cb-b5b4-7aa327ce42d0","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:25:22.433Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"5ee4fddb-dfe0-4b88-9f8a-7550f7506bfa","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:25:22.433Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"5ee4fddb-dfe0-4b88-9f8a-7550f7506bfa","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:25:22.433Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"5ee4fddb-dfe0-4b88-9f8a-7550f7506bfa","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:25:22.433Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"5ee4fddb-dfe0-4b88-9f8a-7550f7506bfa"}
{"level":"info","ts":"2026-11-03T15:25:22.433Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"5ee4fddb-dfe0-4b88-9f8a-7550f7506bfa","RayCluster":"stop-lwhxj","State":""}
{"level":"info","ts":"2026-11-03T15:25:22.433Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"5ee4fddb-dfe0-4b88-9f8a-7550f7506bfa","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:25:25.434Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"0636ecf8-3504-4ec9-9a5d-abac941db8cd","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:25:25.434Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"0636ecf8-3504-4ec9-9a5d-abac941db8cd","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:25:25.434Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"0636ecf8-3504-4ec9-9a5d-abac941db8cd","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:25:25.434Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"0636ecf8-3504-4ec9-9a5d-abac941db8cd"}
{"level":"info","ts":"2026-11-03T15:25:25.434Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"0636ecf8-3504-4ec9-9a5d-abac941db8cd","RayCluster":"stop-lwhxj","State":""}
{"level":"info","ts":"2026-11-03T15:25:25.434Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"0636ecf8-3504-4ec9-9a5d-abac941db8cd","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:25:28.436Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"83c02c32-b4a5-4176-83b6-a1d030be5f5d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:25:28.436Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"83c02c32-b4a5-4176-83b6-a1d030be5f5d","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:25:28.436Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"83c02c32-b4a5-4176-83b6-a1d030be5f5d","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:25:28.436Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"83c02c32-b4a5-4176-83b6-a1d030be5f5d"}
{"level":"info","ts":"2026-11-03T15:25:28.436Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"83c02c32-b4a5-4176-83b6-a1d030be5f5d","RayCluster":"stop-lwhxj","State":""}
{"level":"info","ts":"2026-11-03T15:25:28.436Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"83c02c32-b4a5-4176-83b6-a1d030be5f5d","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:25:29.192Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"1ef92fe8-bcec-4a38-94a1-4e5b4ab9bb3e","cluster":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:25:31.436Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"fc4fc971-3523-4423-b903-07c608a83254","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:25:31.436Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"fc4fc971-3523-4423-b903-07c608a83254","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:25:31.436Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"fc4fc971-3523-4423-b903-07c608a83254","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:25:31.437Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"fc4fc971-3523-4423-b903-07c608a83254"}
{"level":"info","ts":"2026-11-03T15:25:31.437Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"fc4fc971-3523-4423-b903-07c608a83254","RayCluster":"stop-lwhxj","State":""}
{"level":"info","ts":"2026-11-03T15:25:31.437Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"fc4fc971-3523-4423-b903-07c608a83254","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:25:34.438Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"28f14de5-3628-42a8-8c26-1df70045a4b0","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:25:34.438Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"28f14de5-3628-42a8-8c26-1df70045a4b0","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:25:34.438Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"28f14de5-3628-42a8-8c26-1df70045a4b0","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:25:34.438Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"28f14de5-3628-42a8-8c26-1df70045a4b0"}
{"level":"info","ts":"2026-11-03T15:25:34.438Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"28f14de5-3628-42a8-8c26-1df70045a4b0","RayCluster":"stop-lwhxj","State":""}
{"level":"info","ts":"2026-11-03T15:25:34.438Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"28f14de5-3628-42a8-8c26-1df70045a4b0","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:25:37.438Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"0b6ba882-1796-4401-aeca-16e59acd8702","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:25:37.439Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"0b6ba882-1796-4401-aeca-16e59acd8702","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:25:37.439Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"0b6ba882-1796-4401-aeca-16e59acd8702","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:25:37.439Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"0b6ba882-1796-4401-aeca-16e59acd8702"}
{"level":"info","ts":"2026-11-03T15:25:37.439Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"0b6ba882-1796-4401-aeca-16e59acd8702","RayCluster":"stop-lwhxj","State":""}
{"level":"info","ts":"2026-11-03T15:25:37.439Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"0b6ba882-1796-4401-aeca-16e59acd8702","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:25:40.440Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"8454536e-d4e4-40f4-8efc-aaf6e13578a0","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:25:40.440Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"8454536e-d4e4-40f4-8efc-aaf6e13578a0","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:25:40.440Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"8454536e-d4e4-40f4-8efc-aaf6e13578a0","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:25:40.440Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"8454536e-d4e4-40f4-8efc-aaf6e13578a0"}
{"level":"info","ts":"2026-11-03T15:25:40.440Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"8454536e-d4e4-40f4-8efc-aaf6e13578a0","RayCluster":"stop-lwhxj","State":""}
{"level":"info","ts":"2026-11-03T15:25:40.440Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"8454536e-d4e4-40f4-8efc-aaf6e13578a0","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:25:43.440Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"28a5fa37-9ad8-4631-bab1-0a7333fbcdba","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:25:43.440Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"28a5fa37-9ad8-4631-bab1-0a7333fbcdba","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:25:43.440Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"28a5fa37-9ad8-4631-bab1-0a7333fbcdba","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:25:43.440Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"28a5fa37-9ad8-4631-bab1-0a7333fbcdba"}
{"level":"info","ts":"2026-11-03T15:25:43.440Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"28a5fa37-9ad8-4631-bab1-0a7333fbcdba","RayCluster":"stop-lwhxj","State":""}
{"level":"info","ts":"2026-11-03T15:25:43.440Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"28a5fa37-9ad8-4631-bab1-0a7333fbcdba","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:25:46.441Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"f9ff9117-7d71-472f-b49a-254324070869","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:25:46.441Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"f9ff9117-7d71-472f-b49a-254324070869","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:25:46.441Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"f9ff9117-7d71-472f-b49a-254324070869","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:25:46.441Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"f9ff9117-7d71-472f-b49a-254324070869"}
{"level":"info","ts":"2026-11-03T15:25:46.441Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"f9ff9117-7d71-472f-b49a-254324070869","RayCluster":"stop-lwhxj","State":""}
{"level":"info","ts":"2026-11-03T15:25:46.441Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"f9ff9117-7d71-472f-b49a-254324070869","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:25:49.442Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"0c495157-87a9-487e-949a-06c690645789","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:25:49.442Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"0c495157-87a9-487e-949a-06c690645789","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:25:49.442Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"0c495157-87a9-487e-949a-06c690645789","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:25:49.443Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"0c495157-87a9-487e-949a-06c690645789"}
{"level":"info","ts":"2026-11-03T15:25:49.443Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"0c495157-87a9-487e-949a-06c690645789","RayCluster":"stop-lwhxj","State":""}
{"level":"info","ts":"2026-11-03T15:25:49.443Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"0c495157-87a9-487e-949a-06c690645789","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:25:52.443Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"b3d59ec3-3e2e-4ff6-afb7-0cd167c92c73","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:25:52.443Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"b3d59ec3-3e2e-4ff6-afb7-0cd167c92c73","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:25:52.443Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"b3d59ec3-3e2e-4ff6-afb7-0cd167c92c73","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:25:52.444Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"b3d59ec3-3e2e-4ff6-afb7-0cd167c92c73"}
{"level":"info","ts":"2026-11-03T15:25:52.444Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"b3d59ec3-3e2e-4ff6-afb7-0cd167c92c73","RayCluster":"stop-lwhxj","State":""}
{"level":"info","ts":"2026-11-03T15:25:52.444Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"b3d59ec3-3e2e-4ff6-afb7-0cd167c92c73","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:25:55.444Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"7dbd0905-2732-4683-ad4c-ac41ea16b614","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:25:55.444Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"7dbd0905-2732-4683-ad4c-ac41ea16b614","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:25:55.444Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"7dbd0905-2732-4683-ad4c-ac41ea16b614","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:25:55.445Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"7dbd0905-2732-4683-ad4c-ac41ea16b614"}
{"level":"info","ts":"2026-11-03T15:25:55.445Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"7dbd0905-2732-4683-ad4c-ac41ea16b614","RayCluster":"stop-lwhxj","State":""}
{"level":"info","ts":"2026-11-03T15:25:55.445Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"7dbd0905-2732-4683-ad4c-ac41ea16b614","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:25:58.445Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"77d9a663-249c-45c3-a6fc-9dcfbc497738","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:25:58.445Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"77d9a663-249c-45c3-a6fc-9dcfbc497738","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:25:58.445Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"77d9a663-249c-45c3-a6fc-9dcfbc497738","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:25:58.445Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"77d9a663-249c-45c3-a6fc-9dcfbc497738"}
{"level":"info","ts":"2026-11-03T15:25:58.445Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"77d9a663-249c-45c3-a6fc-9dcfbc497738","RayCluster":"stop-lwhxj","State":""}
{"level":"info","ts":"2026-11-03T15:25:58.445Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"77d9a663-249c-45c3-a6fc-9dcfbc497738","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:26:01.446Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"8bbd47fa-ac65-4ef5-8d59-39d9ce0e7376","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:26:01.447Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"8bbd47fa-ac65-4ef5-8d59-39d9ce0e7376","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:26:01.447Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"8bbd47fa-ac65-4ef5-8d59-39d9ce0e7376","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:26:01.447Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"8bbd47fa-ac65-4ef5-8d59-39d9ce0e7376"}
{"level":"info","ts":"2026-11-03T15:26:01.447Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"8bbd47fa-ac65-4ef5-8d59-39d9ce0e7376","RayCluster":"stop-lwhxj","State":""}
{"level":"info","ts":"2026-11-03T15:26:01.447Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"8bbd47fa-ac65-4ef5-8d59-39d9ce0e7376","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:26:04.448Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"e62c795c-28b7-4348-a8d1-66584b1982e6","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:26:04.448Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"e62c795c-28b7-4348-a8d1-66584b1982e6","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:26:04.448Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"e62c795c-28b7-4348-a8d1-66584b1982e6","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:26:04.448Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"e62c795c-28b7-4348-a8d1-66584b1982e6"}
{"level":"info","ts":"2026-11-03T15:26:04.448Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"e62c795c-28b7-4348-a8d1-66584b1982e6","RayCluster":"stop-lwhxj","State":""}
{"level":"info","ts":"2026-11-03T15:26:04.448Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"e62c795c-28b7-4348-a8d1-66584b1982e6","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:26:07.449Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"59477833-f9c3-423e-975a-c2b4b7885d3a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:26:07.449Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"59477833-f9c3-423e-975a-c2b4b7885d3a","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:26:07.449Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"59477833-f9c3-423e-975a-c2b4b7885d3a","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:26:07.449Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"59477833-f9c3-423e-975a-c2b4b7885d3a"}
{"level":"info","ts":"2026-11-03T15:26:07.449Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"59477833-f9c3-423e-975a-c2b4b7885d3a","RayCluster":"stop-lwhxj","State":""}
{"level":"info","ts":"2026-11-03T15:26:07.449Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"59477833-f9c3-423e-975a-c2b4b7885d3a","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:26:10.450Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"a7735dda-33ef-4b4b-b2d3-cb60a7299643","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:26:10.450Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"a7735dda-33ef-4b4b-b2d3-cb60a7299643","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:26:10.450Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"a7735dda-33ef-4b4b-b2d3-cb60a7299643","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:26:10.450Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"a7735dda-33ef-4b4b-b2d3-cb60a7299643"}
{"level":"info","ts":"2026-11-03T15:26:10.450Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"a7735dda-33ef-4b4b-b2d3-cb60a7299643","RayCluster":"stop-lwhxj","State":""}
{"level":"info","ts":"2026-11-03T15:26:10.450Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"a7735dda-33ef-4b4b-b2d3-cb60a7299643","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:26:13.451Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"ed6bcd3b-0e2d-4246-8559-257ec36b1759","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:26:13.451Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"ed6bcd3b-0e2d-4246-8559-257ec36b1759","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:26:13.451Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"ed6bcd3b-0e2d-4246-8559-257ec36b1759","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:26:13.451Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"ed6bcd3b-0e2d-4246-8559-257ec36b1759"}
{"level":"info","ts":"2026-11-03T15:26:13.451Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"ed6bcd3b-0e2d-4246-8559-257ec36b1759","RayCluster":"stop-lwhxj","State":""}
{"level":"info","ts":"2026-11-03T15:26:13.451Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"ed6bcd3b-0e2d-4246-8559-257ec36b1759","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:26:16.452Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"b706cef5-686a-4d3c-aa0a-add5d4e16012","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:26:16.452Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"b706cef5-686a-4d3c-aa0a-add5d4e16012","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:26:16.452Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"b706cef5-686a-4d3c-aa0a-add5d4e16012","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:26:16.452Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"b706cef5-686a-4d3c-aa0a-add5d4e16012"}
{"level":"info","ts":"2026-11-03T15:26:16.452Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"b706cef5-686a-4d3c-aa0a-add5d4e16012","RayCluster":"stop-lwhxj","State":""}
{"level":"info","ts":"2026-11-03T15:26:16.452Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"b706cef5-686a-4d3c-aa0a-add5d4e16012","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:26:18.669Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"487641c1-4e8c-4029-9d2b-6983399511cc"}
{"level":"info","ts":"2026-11-03T15:26:18.669Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"487641c1-4e8c-4029-9d2b-6983399511cc"}
{"level":"info","ts":"2026-11-03T15:26:18.669Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"487641c1-4e8c-4029-9d2b-6983399511cc","1 head service found":"stop-lwhxj-head-svc"}
{"level":"info","ts":"2026-11-03T15:26:18.669Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"487641c1-4e8c-4029-9d2b-6983399511cc","Found 1 head Pod":"stop-lwhxj-head-brfn5","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:26:18.669Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"487641c1-4e8c-4029-9d2b-6983399511cc","head Pod":"stop-lwhxj-head-brfn5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-lwhxj-head-brfn5. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:26:18.669Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"487641c1-4e8c-4029-9d2b-6983399511cc","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:26:18.669Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"487641c1-4e8c-4029-9d2b-6983399511cc","worker Pod":"stop-lwhxj-small-group-worker-zxh4p","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-lwhxj-small-group-worker-zxh4p. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:26:18.669Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"487641c1-4e8c-4029-9d2b-6983399511cc","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:26:18.669Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"487641c1-4e8c-4029-9d2b-6983399511cc","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:26:18.669Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"487641c1-4e8c-4029-9d2b-6983399511cc","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:26:18.669Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"487641c1-4e8c-4029-9d2b-6983399511cc","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:26:18.669Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"487641c1-4e8c-4029-9d2b-6983399511cc","podName":"stop-lwhxj-head-brfn5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:26:18.669Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"487641c1-4e8c-4029-9d2b-6983399511cc","podName":"stop-lwhxj-head-brfn5","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:26:18.669Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"487641c1-4e8c-4029-9d2b-6983399511cc","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:26:18.669Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"487641c1-4e8c-4029-9d2b-6983399511cc","seconds":300}
{"level":"info","ts":"2026-11-03T15:26:19.452Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"371f7835-42b5-4ca1-979c-70068861de13","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:26:19.452Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"371f7835-42b5-4ca1-979c-70068861de13","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:26:19.453Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"371f7835-42b5-4ca1-979c-70068861de13","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:26:19.453Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"371f7835-42b5-4ca1-979c-70068861de13"}
{"level":"info","ts":"2026-11-03T15:26:19.453Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"371f7835-42b5-4ca1-979c-70068861de13","RayCluster":"stop-lwhxj","State":""}
{"level":"info","ts":"2026-11-03T15:26:19.453Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"371f7835-42b5-4ca1-979c-70068861de13","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:26:19.723Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"4655e2b3-0fb0-4a25-9b52-8f951a6e446d"}
{"level":"info","ts":"2026-11-03T15:26:19.723Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"4655e2b3-0fb0-4a25-9b52-8f951a6e446d"}
{"level":"info","ts":"2026-11-03T15:26:19.723Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"4655e2b3-0fb0-4a25-9b52-8f951a6e446d","1 head service found":"stop-lwhxj-head-svc"}
{"level":"info","ts":"2026-11-03T15:26:19.723Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"4655e2b3-0fb0-4a25-9b52-8f951a6e446d","Found 1 head Pod":"stop-lwhxj-head-brfn5","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:26:19.723Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"4655e2b3-0fb0-4a25-9b52-8f951a6e446d","head Pod":"stop-lwhxj-head-brfn5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-lwhxj-head-brfn5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:26:19.723Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"4655e2b3-0fb0-4a25-9b52-8f951a6e446d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:26:19.723Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"4655e2b3-0fb0-4a25-9b52-8f951a6e446d","worker Pod":"stop-lwhxj-small-group-worker-zxh4p","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-lwhxj-small-group-worker-zxh4p. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:26:19.723Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"4655e2b3-0fb0-4a25-9b52-8f951a6e446d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:26:19.723Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"4655e2b3-0fb0-4a25-9b52-8f951a6e446d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:26:19.723Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"4655e2b3-0fb0-4a25-9b52-8f951a6e446d","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:26:19.723Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"4655e2b3-0fb0-4a25-9b52-8f951a6e446d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:26:19.723Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"4655e2b3-0fb0-4a25-9b52-8f951a6e446d","podName":"stop-lwhxj-small-group-worker-zxh4p","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:26:19.723Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"4655e2b3-0fb0-4a25-9b52-8f951a6e446d","podName":"stop-lwhxj-small-group-worker-zxh4p","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:26:19.723Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"4655e2b3-0fb0-4a25-9b52-8f951a6e446d","name":"stop-lwhxj","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:26:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:26:19.785Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"395bbb70-4f3a-432b-956e-3f57db10dc6d","cluster":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:26:19.785Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"3339b6f4-3ec8-4b13-ae8d-5fea56858d58","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:26:19.785Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"3339b6f4-3ec8-4b13-ae8d-5fea56858d58","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:26:19.785Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"3339b6f4-3ec8-4b13-ae8d-5fea56858d58","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:26:19.785Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"3339b6f4-3ec8-4b13-ae8d-5fea56858d58"}
{"level":"info","ts":"2026-11-03T15:26:19.785Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"3339b6f4-3ec8-4b13-ae8d-5fea56858d58","RayCluster":"stop-lwhxj","State":""}
{"level":"info","ts":"2026-11-03T15:26:19.785Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"3339b6f4-3ec8-4b13-ae8d-5fea56858d58","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:24:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:26:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:26:19.785Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"3339b6f4-3ec8-4b13-ae8d-5fea56858d58","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:26:19.847Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"713008c0-a84e-4047-a826-e43a3e08c52a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:26:19.847Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"713008c0-a84e-4047-a826-e43a3e08c52a","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:26:19.847Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"713008c0-a84e-4047-a826-e43a3e08c52a","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:26:19.847Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"713008c0-a84e-4047-a826-e43a3e08c52a"}
{"level":"info","ts":"2026-11-03T15:26:19.847Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"713008c0-a84e-4047-a826-e43a3e08c52a","RayCluster":"stop-lwhxj","State":""}
{"level":"info","ts":"2026-11-03T15:26:19.847Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"713008c0-a84e-4047-a826-e43a3e08c52a","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:26:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:26:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:26:21.785Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"683266e0-edf3-4cd9-b68a-fb1c0bcd5bfe"}
{"level":"info","ts":"2026-11-03T15:26:21.785Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"683266e0-edf3-4cd9-b68a-fb1c0bcd5bfe"}
{"level":"info","ts":"2026-11-03T15:26:21.785Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"683266e0-edf3-4cd9-b68a-fb1c0bcd5bfe","1 head service found":"stop-lwhxj-head-svc"}
{"level":"info","ts":"2026-11-03T15:26:21.785Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"683266e0-edf3-4cd9-b68a-fb1c0bcd5bfe","Found 1 head Pod":"stop-lwhxj-head-brfn5","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:26:21.785Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"683266e0-edf3-4cd9-b68a-fb1c0bcd5bfe","head Pod":"stop-lwhxj-head-brfn5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-lwhxj-head-brfn5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:26:21.785Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"683266e0-edf3-4cd9-b68a-fb1c0bcd5bfe","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:26:21.785Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"683266e0-edf3-4cd9-b68a-fb1c0bcd5bfe","worker Pod":"stop-lwhxj-small-group-worker-zxh4p","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-lwhxj-small-group-worker-zxh4p. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:26:21.785Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"683266e0-edf3-4cd9-b68a-fb1c0bcd5bfe","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:26:21.785Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"683266e0-edf3-4cd9-b68a-fb1c0bcd5bfe","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:26:21.785Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"683266e0-edf3-4cd9-b68a-fb1c0bcd5bfe","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:26:21.785Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"683266e0-edf3-4cd9-b68a-fb1c0bcd5bfe","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:26:21.786Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"683266e0-edf3-4cd9-b68a-fb1c0bcd5bfe","podName":"stop-lwhxj-head-brfn5","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:24:57Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:26:21.786Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"683266e0-edf3-4cd9-b68a-fb1c0bcd5bfe","podName":"stop-lwhxj-head-brfn5","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:24:57Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:26:21.786Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"683266e0-edf3-4cd9-b68a-fb1c0bcd5bfe","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:26:21.786Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"683266e0-edf3-4cd9-b68a-fb1c0bcd5bfe","seconds":300}
{"level":"info","ts":"2026-11-03T15:26:22.848Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"b9da23db-fd07-4abb-88d8-d9cbaae74c07","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:26:22.848Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"b9da23db-fd07-4abb-88d8-d9cbaae74c07","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:26:22.848Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"b9da23db-fd07-4abb-88d8-d9cbaae74c07","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:26:22.848Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"b9da23db-fd07-4abb-88d8-d9cbaae74c07"}
{"level":"info","ts":"2026-11-03T15:26:22.848Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"b9da23db-fd07-4abb-88d8-d9cbaae74c07","RayCluster":"stop-lwhxj","State":""}
{"level":"info","ts":"2026-11-03T15:26:22.848Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"b9da23db-fd07-4abb-88d8-d9cbaae74c07","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:26:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:26:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:26:25.849Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"8856b000-47cd-4287-8c99-cf443f8d5496","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:26:25.849Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"8856b000-47cd-4287-8c99-cf443f8d5496","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:26:25.849Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"8856b000-47cd-4287-8c99-cf443f8d5496","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:26:25.849Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"8856b000-47cd-4287-8c99-cf443f8d5496"}
{"level":"info","ts":"2026-11-03T15:26:25.849Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"8856b000-47cd-4287-8c99-cf443f8d5496","RayCluster":"stop-lwhxj","State":""}
{"level":"info","ts":"2026-11-03T15:26:25.849Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"8856b000-47cd-4287-8c99-cf443f8d5496","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:26:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:26:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:26:28.851Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"69708c5a-7c38-41a2-b3c4-858e87944520","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:26:28.851Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"69708c5a-7c38-41a2-b3c4-858e87944520","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:26:28.851Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"69708c5a-7c38-41a2-b3c4-858e87944520","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:26:28.851Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"69708c5a-7c38-41a2-b3c4-858e87944520"}
{"level":"info","ts":"2026-11-03T15:26:28.851Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"69708c5a-7c38-41a2-b3c4-858e87944520","RayCluster":"stop-lwhxj","State":""}
{"level":"info","ts":"2026-11-03T15:26:28.851Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"69708c5a-7c38-41a2-b3c4-858e87944520","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:26:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:26:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:26:29.826Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"cd829174-3556-49ed-875d-d35c2cbfd37a"}
{"level":"info","ts":"2026-11-03T15:26:29.826Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"cd829174-3556-49ed-875d-d35c2cbfd37a"}
{"level":"info","ts":"2026-11-03T15:26:29.826Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"cd829174-3556-49ed-875d-d35c2cbfd37a","1 head service found":"stop-lwhxj-head-svc"}
{"level":"info","ts":"2026-11-03T15:26:29.826Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"cd829174-3556-49ed-875d-d35c2cbfd37a","Found 1 head Pod":"stop-lwhxj-head-brfn5","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:26:29.826Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"cd829174-3556-49ed-875d-d35c2cbfd37a","head Pod":"stop-lwhxj-head-brfn5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-lwhxj-head-brfn5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:26:29.826Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"cd829174-3556-49ed-875d-d35c2cbfd37a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:26:29.826Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"cd829174-3556-49ed-875d-d35c2cbfd37a","worker Pod":"stop-lwhxj-small-group-worker-zxh4p","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-lwhxj-small-group-worker-zxh4p. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:26:29.826Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"cd829174-3556-49ed-875d-d35c2cbfd37a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:26:29.826Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"cd829174-3556-49ed-875d-d35c2cbfd37a","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:26:29.826Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"cd829174-3556-49ed-875d-d35c2cbfd37a","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:26:29.826Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"cd829174-3556-49ed-875d-d35c2cbfd37a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:26:29.826Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"cd829174-3556-49ed-875d-d35c2cbfd37a","podName":"stop-lwhxj-small-group-worker-zxh4p","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:26:29.826Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"cd829174-3556-49ed-875d-d35c2cbfd37a","podName":"stop-lwhxj-small-group-worker-zxh4p","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:26:29.827Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"cd829174-3556-49ed-875d-d35c2cbfd37a","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:26:29.827Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"cd829174-3556-49ed-875d-d35c2cbfd37a","seconds":300}
{"level":"info","ts":"2026-11-03T15:26:30.727Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"15bd1d95-9013-401b-a334-09d8dfbb5635"}
{"level":"info","ts":"2026-11-03T15:26:30.727Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"15bd1d95-9013-401b-a334-09d8dfbb5635"}
{"level":"info","ts":"2026-11-03T15:26:30.727Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"15bd1d95-9013-401b-a334-09d8dfbb5635","1 head service found":"stop-lwhxj-head-svc"}
{"level":"info","ts":"2026-11-03T15:26:30.727Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"15bd1d95-9013-401b-a334-09d8dfbb5635","Found 1 head Pod":"stop-lwhxj-head-brfn5","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:26:30.727Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"15bd1d95-9013-401b-a334-09d8dfbb5635","head Pod":"stop-lwhxj-head-brfn5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-lwhxj-head-brfn5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:26:30.727Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"15bd1d95-9013-401b-a334-09d8dfbb5635","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:26:30.727Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"15bd1d95-9013-401b-a334-09d8dfbb5635","worker Pod":"stop-lwhxj-small-group-worker-zxh4p","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-lwhxj-small-group-worker-zxh4p. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:26:30.727Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"15bd1d95-9013-401b-a334-09d8dfbb5635","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:26:30.727Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"15bd1d95-9013-401b-a334-09d8dfbb5635","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:26:30.727Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"15bd1d95-9013-401b-a334-09d8dfbb5635","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:26:30.727Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"15bd1d95-9013-401b-a334-09d8dfbb5635","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:26:30.727Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"15bd1d95-9013-401b-a334-09d8dfbb5635","podName":"stop-lwhxj-small-group-worker-zxh4p","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:26:30.727Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"15bd1d95-9013-401b-a334-09d8dfbb5635","podName":"stop-lwhxj-small-group-worker-zxh4p","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:26:30.728Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"15bd1d95-9013-401b-a334-09d8dfbb5635","name":"stop-lwhxj","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:26:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:26:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:26:30Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:26:30.790Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"c33c1ace-d7e2-415c-9561-9def63234d25","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:26:30.790Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"c33c1ace-d7e2-415c-9561-9def63234d25","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:26:30.790Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"c33c1ace-d7e2-415c-9561-9def63234d25","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:26:30.790Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"c4da9b8f-1e8b-4cd2-8030-a93427c6e78e","cluster":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:26:30.790Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"c33c1ace-d7e2-415c-9561-9def63234d25"}
{"level":"info","ts":"2026-11-03T15:26:30.790Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"c33c1ace-d7e2-415c-9561-9def63234d25","RayCluster":"stop-lwhxj","State":""}
{"level":"info","ts":"2026-11-03T15:26:30.790Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"c33c1ace-d7e2-415c-9561-9def63234d25","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:26:19Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:26:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:26:30Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:26:30.790Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"c33c1ace-d7e2-415c-9561-9def63234d25","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:26:30.828Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"b32ec471-83e3-451e-bec7-1ce2569d28a6"}
{"level":"info","ts":"2026-11-03T15:26:30.828Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"b32ec471-83e3-451e-bec7-1ce2569d28a6"}
{"level":"info","ts":"2026-11-03T15:26:30.828Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"b32ec471-83e3-451e-bec7-1ce2569d28a6","1 head service found":"stop-lwhxj-head-svc"}
{"level":"info","ts":"2026-11-03T15:26:30.828Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"b32ec471-83e3-451e-bec7-1ce2569d28a6","Found 1 head Pod":"stop-lwhxj-head-brfn5","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:26:30.828Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"b32ec471-83e3-451e-bec7-1ce2569d28a6","head Pod":"stop-lwhxj-head-brfn5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-lwhxj-head-brfn5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:26:30.828Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"b32ec471-83e3-451e-bec7-1ce2569d28a6","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:26:30.828Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"b32ec471-83e3-451e-bec7-1ce2569d28a6","worker Pod":"stop-lwhxj-small-group-worker-zxh4p","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-lwhxj-small-group-worker-zxh4p. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:26:30.828Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"b32ec471-83e3-451e-bec7-1ce2569d28a6","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:26:30.828Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"b32ec471-83e3-451e-bec7-1ce2569d28a6","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:26:30.828Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"b32ec471-83e3-451e-bec7-1ce2569d28a6","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:26:30.828Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"b32ec471-83e3-451e-bec7-1ce2569d28a6","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:26:30.828Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"b32ec471-83e3-451e-bec7-1ce2569d28a6","podName":"stop-lwhxj-small-group-worker-zxh4p","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:26:30.828Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"b32ec471-83e3-451e-bec7-1ce2569d28a6","podName":"stop-lwhxj-small-group-worker-zxh4p","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:26:30.828Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"b32ec471-83e3-451e-bec7-1ce2569d28a6","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:26:30.828Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"b32ec471-83e3-451e-bec7-1ce2569d28a6","seconds":300}
{"level":"info","ts":"2026-11-03T15:26:30.852Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"0e7132de-cd44-4b47-99aa-ecbc433f6aee","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:26:30.852Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"0e7132de-cd44-4b47-99aa-ecbc433f6aee","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:26:30.852Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"0e7132de-cd44-4b47-99aa-ecbc433f6aee","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:26:30.852Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"0e7132de-cd44-4b47-99aa-ecbc433f6aee"}
{"level":"info","ts":"2026-11-03T15:26:30.852Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"0e7132de-cd44-4b47-99aa-ecbc433f6aee","RayCluster":"stop-lwhxj","State":""}
{"level":"info","ts":"2026-11-03T15:26:30.852Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"0e7132de-cd44-4b47-99aa-ecbc433f6aee","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:26:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:26:30Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:26:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:26:30Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:26:33.853Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"9f4cff17-2427-44a2-94b3-c9f5f4c0da95","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:26:33.853Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"9f4cff17-2427-44a2-94b3-c9f5f4c0da95","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:26:33.853Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"9f4cff17-2427-44a2-94b3-c9f5f4c0da95","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:26:33.853Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"9f4cff17-2427-44a2-94b3-c9f5f4c0da95"}
{"level":"info","ts":"2026-11-03T15:26:33.853Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"9f4cff17-2427-44a2-94b3-c9f5f4c0da95","RayCluster":"stop-lwhxj","State":""}
{"level":"info","ts":"2026-11-03T15:26:33.853Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"9f4cff17-2427-44a2-94b3-c9f5f4c0da95","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:26:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:26:30Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:26:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:26:30Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:26:36.853Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"655cb6a3-b4d3-4d18-8ade-9e3fe19383d9","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:26:36.853Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"655cb6a3-b4d3-4d18-8ade-9e3fe19383d9","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:26:36.853Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"655cb6a3-b4d3-4d18-8ade-9e3fe19383d9","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:26:36.854Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"655cb6a3-b4d3-4d18-8ade-9e3fe19383d9"}
{"level":"info","ts":"2026-11-03T15:26:36.854Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"655cb6a3-b4d3-4d18-8ade-9e3fe19383d9","RayCluster":"stop-lwhxj","State":""}
{"level":"info","ts":"2026-11-03T15:26:36.854Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"655cb6a3-b4d3-4d18-8ade-9e3fe19383d9","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:26:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:26:30Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:26:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:26:30Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:26:39.854Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"3445bc76-a0b5-401a-a21e-70870cebac88","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:26:39.854Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"3445bc76-a0b5-401a-a21e-70870cebac88","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:26:39.854Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"3445bc76-a0b5-401a-a21e-70870cebac88","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:26:39.855Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"3445bc76-a0b5-401a-a21e-70870cebac88"}
{"level":"info","ts":"2026-11-03T15:26:39.855Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"3445bc76-a0b5-401a-a21e-70870cebac88","RayCluster":"stop-lwhxj","State":""}
{"level":"info","ts":"2026-11-03T15:26:39.855Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"3445bc76-a0b5-401a-a21e-70870cebac88","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:26:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:26:30Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:26:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:26:30Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:26:39.866Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"f5c104b0-9e16-4ddc-80ec-f7146ffbb239"}
{"level":"info","ts":"2026-11-03T15:26:39.866Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"f5c104b0-9e16-4ddc-80ec-f7146ffbb239"}
{"level":"info","ts":"2026-11-03T15:26:39.866Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"f5c104b0-9e16-4ddc-80ec-f7146ffbb239","1 head service found":"stop-lwhxj-head-svc"}
{"level":"info","ts":"2026-11-03T15:26:39.866Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"f5c104b0-9e16-4ddc-80ec-f7146ffbb239","Found 1 head Pod":"stop-lwhxj-head-brfn5","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:26:39.866Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"f5c104b0-9e16-4ddc-80ec-f7146ffbb239","head Pod":"stop-lwhxj-head-brfn5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-lwhxj-head-brfn5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:26:39.866Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"f5c104b0-9e16-4ddc-80ec-f7146ffbb239","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:26:39.866Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"f5c104b0-9e16-4ddc-80ec-f7146ffbb239","worker Pod":"stop-lwhxj-small-group-worker-zxh4p","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-lwhxj-small-group-worker-zxh4p. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:26:39.866Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"f5c104b0-9e16-4ddc-80ec-f7146ffbb239","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:26:39.866Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"f5c104b0-9e16-4ddc-80ec-f7146ffbb239","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:26:39.866Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"f5c104b0-9e16-4ddc-80ec-f7146ffbb239","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:26:39.866Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"f5c104b0-9e16-4ddc-80ec-f7146ffbb239","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:26:39.866Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"f5c104b0-9e16-4ddc-80ec-f7146ffbb239","podName":"stop-lwhxj-small-group-worker-zxh4p","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:26:39.866Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"f5c104b0-9e16-4ddc-80ec-f7146ffbb239","podName":"stop-lwhxj-small-group-worker-zxh4p","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:26:39.866Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"f5c104b0-9e16-4ddc-80ec-f7146ffbb239","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:26:39.866Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"f5c104b0-9e16-4ddc-80ec-f7146ffbb239","seconds":300}
{"level":"info","ts":"2026-11-03T15:26:40.872Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"fd81c413-219b-4d89-9d96-b4d12fde8f1f"}
{"level":"info","ts":"2026-11-03T15:26:40.872Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"fd81c413-219b-4d89-9d96-b4d12fde8f1f"}
{"level":"info","ts":"2026-11-03T15:26:40.872Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"fd81c413-219b-4d89-9d96-b4d12fde8f1f","1 head service found":"stop-lwhxj-head-svc"}
{"level":"info","ts":"2026-11-03T15:26:40.872Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"fd81c413-219b-4d89-9d96-b4d12fde8f1f","Found 1 head Pod":"stop-lwhxj-head-brfn5","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:26:40.872Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"fd81c413-219b-4d89-9d96-b4d12fde8f1f","head Pod":"stop-lwhxj-head-brfn5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-lwhxj-head-brfn5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:26:40.872Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"fd81c413-219b-4d89-9d96-b4d12fde8f1f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:26:40.872Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"fd81c413-219b-4d89-9d96-b4d12fde8f1f","worker Pod":"stop-lwhxj-small-group-worker-zxh4p","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-lwhxj-small-group-worker-zxh4p. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:26:40.872Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"fd81c413-219b-4d89-9d96-b4d12fde8f1f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:26:40.872Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"fd81c413-219b-4d89-9d96-b4d12fde8f1f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:26:40.872Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"fd81c413-219b-4d89-9d96-b4d12fde8f1f","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:26:40.872Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"fd81c413-219b-4d89-9d96-b4d12fde8f1f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:26:40.872Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"fd81c413-219b-4d89-9d96-b4d12fde8f1f","podName":"stop-lwhxj-small-group-worker-zxh4p","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:24:57Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:26:40.872Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"fd81c413-219b-4d89-9d96-b4d12fde8f1f","podName":"stop-lwhxj-small-group-worker-zxh4p","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:24:57Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:26:40.873Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"fd81c413-219b-4d89-9d96-b4d12fde8f1f","name":"stop-lwhxj","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:26:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:26:30Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:26:40Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:26:30Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:26:40.935Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"13835bf9-f41b-49ab-891a-bc69f7f14a3f","cluster":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:26:40.935Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"da853a06-a72d-4183-9cba-cea73fc64048","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:26:40.935Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"da853a06-a72d-4183-9cba-cea73fc64048","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:26:40.935Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"da853a06-a72d-4183-9cba-cea73fc64048","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:26:40.935Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"da853a06-a72d-4183-9cba-cea73fc64048"}
{"level":"info","ts":"2026-11-03T15:26:40.935Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"da853a06-a72d-4183-9cba-cea73fc64048","RayCluster":"stop-lwhxj","State":""}
{"level":"info","ts":"2026-11-03T15:26:40.935Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"da853a06-a72d-4183-9cba-cea73fc64048","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:26:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:26:30Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:26:40Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:26:30Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:26:40.935Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"da853a06-a72d-4183-9cba-cea73fc64048","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:26:40.996Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"a13ae4cd-754b-4a68-93b9-96a94d52e0b4","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:26:40.996Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"a13ae4cd-754b-4a68-93b9-96a94d52e0b4","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:26:40.996Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"a13ae4cd-754b-4a68-93b9-96a94d52e0b4","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:26:40.997Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"a13ae4cd-754b-4a68-93b9-96a94d52e0b4"}
{"level":"info","ts":"2026-11-03T15:26:40.997Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"a13ae4cd-754b-4a68-93b9-96a94d52e0b4","RayCluster":"stop-lwhxj","State":""}
{"level":"info","ts":"2026-11-03T15:26:40.997Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"a13ae4cd-754b-4a68-93b9-96a94d52e0b4","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:26:40Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:26:30Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:26:40Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:26:30Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:26:42.935Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"59a2ad78-1daf-4c64-b87e-aab77d430138"}
{"level":"info","ts":"2026-11-03T15:26:42.935Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"59a2ad78-1daf-4c64-b87e-aab77d430138"}
{"level":"info","ts":"2026-11-03T15:26:42.935Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"59a2ad78-1daf-4c64-b87e-aab77d430138","1 head service found":"stop-lwhxj-head-svc"}
{"level":"info","ts":"2026-11-03T15:26:42.935Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"59a2ad78-1daf-4c64-b87e-aab77d430138","Found 1 head Pod":"stop-lwhxj-head-brfn5","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:26:42.935Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"59a2ad78-1daf-4c64-b87e-aab77d430138","head Pod":"stop-lwhxj-head-brfn5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-lwhxj-head-brfn5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:26:42.935Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"59a2ad78-1daf-4c64-b87e-aab77d430138","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:26:42.935Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"59a2ad78-1daf-4c64-b87e-aab77d430138","worker Pod":"stop-lwhxj-small-group-worker-zxh4p","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-lwhxj-small-group-worker-zxh4p. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:26:42.935Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"59a2ad78-1daf-4c64-b87e-aab77d430138","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:26:42.935Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"59a2ad78-1daf-4c64-b87e-aab77d430138","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:26:42.935Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"59a2ad78-1daf-4c64-b87e-aab77d430138","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:26:42.935Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"59a2ad78-1daf-4c64-b87e-aab77d430138","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:26:42.935Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"59a2ad78-1daf-4c64-b87e-aab77d430138","podName":"stop-lwhxj-small-group-worker-zxh4p","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:24:57Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:26:42.935Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"59a2ad78-1daf-4c64-b87e-aab77d430138","podName":"stop-lwhxj-small-group-worker-zxh4p","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:24:57Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:26:42.935Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"59a2ad78-1daf-4c64-b87e-aab77d430138","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:26:42.935Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"59a2ad78-1daf-4c64-b87e-aab77d430138","seconds":300}
{"level":"info","ts":"2026-11-03T15:26:43.997Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"c01620ca-d062-4c94-a72f-7a96872f1906","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:26:43.998Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"c01620ca-d062-4c94-a72f-7a96872f1906","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:26:43.998Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"c01620ca-d062-4c94-a72f-7a96872f1906","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:26:43.998Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"c01620ca-d062-4c94-a72f-7a96872f1906"}
{"level":"info","ts":"2026-11-03T15:26:43.998Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"c01620ca-d062-4c94-a72f-7a96872f1906","RayCluster":"stop-lwhxj","State":""}
{"level":"info","ts":"2026-11-03T15:26:43.998Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"c01620ca-d062-4c94-a72f-7a96872f1906","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:26:40Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:26:30Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:26:40Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:26:30Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:26:46.999Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"b318c09c-30d0-4b2d-8283-5dd1095ae7b4","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:26:46.999Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"b318c09c-30d0-4b2d-8283-5dd1095ae7b4","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:26:46.999Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"b318c09c-30d0-4b2d-8283-5dd1095ae7b4","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:26:46.999Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"b318c09c-30d0-4b2d-8283-5dd1095ae7b4"}
{"level":"info","ts":"2026-11-03T15:26:46.999Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"b318c09c-30d0-4b2d-8283-5dd1095ae7b4","RayCluster":"stop-lwhxj","State":""}
{"level":"info","ts":"2026-11-03T15:26:46.999Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"b318c09c-30d0-4b2d-8283-5dd1095ae7b4","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:26:40Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:26:30Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:26:40Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:26:30Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:26:49.999Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"aadbd835-e618-4678-a3b4-30582a593092","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:26:49.999Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"aadbd835-e618-4678-a3b4-30582a593092","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:26:49.999Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"aadbd835-e618-4678-a3b4-30582a593092","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:26:50.000Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"aadbd835-e618-4678-a3b4-30582a593092"}
{"level":"info","ts":"2026-11-03T15:26:50.000Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"aadbd835-e618-4678-a3b4-30582a593092","RayCluster":"stop-lwhxj","State":""}
{"level":"info","ts":"2026-11-03T15:26:50.000Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"aadbd835-e618-4678-a3b4-30582a593092","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:26:40Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:26:30Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:26:40Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:26:30Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:26:51.881Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"553304d8-76db-4147-86b2-aca307cc595b"}
{"level":"info","ts":"2026-11-03T15:26:51.881Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"553304d8-76db-4147-86b2-aca307cc595b"}
{"level":"info","ts":"2026-11-03T15:26:51.881Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"553304d8-76db-4147-86b2-aca307cc595b","1 head service found":"stop-lwhxj-head-svc"}
{"level":"info","ts":"2026-11-03T15:26:51.881Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"553304d8-76db-4147-86b2-aca307cc595b","Found 1 head Pod":"stop-lwhxj-head-brfn5","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:26:51.881Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"553304d8-76db-4147-86b2-aca307cc595b","head Pod":"stop-lwhxj-head-brfn5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-lwhxj-head-brfn5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:26:51.881Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"553304d8-76db-4147-86b2-aca307cc595b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:26:51.881Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"553304d8-76db-4147-86b2-aca307cc595b","worker Pod":"stop-lwhxj-small-group-worker-zxh4p","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-lwhxj-small-group-worker-zxh4p. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:26:51.881Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"553304d8-76db-4147-86b2-aca307cc595b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:26:51.881Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"553304d8-76db-4147-86b2-aca307cc595b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:26:51.881Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"553304d8-76db-4147-86b2-aca307cc595b","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:26:51.881Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"553304d8-76db-4147-86b2-aca307cc595b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:26:51.882Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"553304d8-76db-4147-86b2-aca307cc595b","name":"stop-lwhxj","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:26:40Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:26:30Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:26:51Z","stateTransitionTimes":{"ready":"2026-11-03T15:26:51Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:26:30Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:26:51Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:26:51.943Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"f64e211d-df4a-45a8-b1dc-384d9178f713","cluster":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:26:51.943Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"701949c9-d5eb-493d-b389-9522c2d7826c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:26:51.943Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"701949c9-d5eb-493d-b389-9522c2d7826c","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:26:51.943Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"701949c9-d5eb-493d-b389-9522c2d7826c","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:26:51.944Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"701949c9-d5eb-493d-b389-9522c2d7826c"}
{"level":"info","ts":"2026-11-03T15:26:52.005Z","logger":"controllers.RayJob","msg":"Created submitter Kubernetes Job for RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"701949c9-d5eb-493d-b389-9522c2d7826c","Kubernetes Job":"stop"}
{"level":"info","ts":"2026-11-03T15:26:52.005Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"701949c9-d5eb-493d-b389-9522c2d7826c","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:26:40Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:26:30Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","dashboardURL":"stop-lwhxj-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:26:40Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:26:30Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:26:52.005Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"701949c9-d5eb-493d-b389-9522c2d7826c","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-11-03T15:26:52.068Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"c10cda2f-80d8-437e-b33b-22a3318e4df6","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:26:52.068Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"c10cda2f-80d8-437e-b33b-22a3318e4df6","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:26:52.068Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"c10cda2f-80d8-437e-b33b-22a3318e4df6","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:26:52.068Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"c10cda2f-80d8-437e-b33b-22a3318e4df6"}
{"level":"info","ts":"2026-11-03T15:26:52.068Z","logger":"controllers.RayJob","msg":"Create a Kubernetes Service","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"c10cda2f-80d8-437e-b33b-22a3318e4df6","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:26:52.168Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"9c680f8b-c55d-4a1a-bb57-4dfbb9aab793","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:26:52.168Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"9c680f8b-c55d-4a1a-bb57-4dfbb9aab793","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:26:52.168Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"9c680f8b-c55d-4a1a-bb57-4dfbb9aab793","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:26:52.168Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"9c680f8b-c55d-4a1a-bb57-4dfbb9aab793"}
{"level":"info","ts":"2026-11-03T15:26:52.168Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"9c680f8b-c55d-4a1a-bb57-4dfbb9aab793","rayCluster":"stop-lwhxj","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:26:53.944Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"e4e8b3f0-2756-4b1f-ad43-2e6d83285be2"}
{"level":"info","ts":"2026-11-03T15:26:53.944Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"e4e8b3f0-2756-4b1f-ad43-2e6d83285be2"}
{"level":"info","ts":"2026-11-03T15:26:53.944Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"e4e8b3f0-2756-4b1f-ad43-2e6d83285be2","1 head service found":"stop-lwhxj-head-svc"}
{"level":"info","ts":"2026-11-03T15:26:53.944Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"e4e8b3f0-2756-4b1f-ad43-2e6d83285be2","Found 1 head Pod":"stop-lwhxj-head-brfn5","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:26:53.944Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"e4e8b3f0-2756-4b1f-ad43-2e6d83285be2","head Pod":"stop-lwhxj-head-brfn5","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod stop-lwhxj-head-brfn5. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:26:53.944Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"e4e8b3f0-2756-4b1f-ad43-2e6d83285be2","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:26:53.944Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"e4e8b3f0-2756-4b1f-ad43-2e6d83285be2","worker Pod":"stop-lwhxj-small-group-worker-zxh4p","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod stop-lwhxj-small-group-worker-zxh4p. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:26:53.944Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"e4e8b3f0-2756-4b1f-ad43-2e6d83285be2","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:26:53.944Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"e4e8b3f0-2756-4b1f-ad43-2e6d83285be2","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:26:53.944Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"e4e8b3f0-2756-4b1f-ad43-2e6d83285be2","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:26:53.944Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"e4e8b3f0-2756-4b1f-ad43-2e6d83285be2","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:26:53.945Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"e4e8b3f0-2756-4b1f-ad43-2e6d83285be2","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:26:53.945Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"e4e8b3f0-2756-4b1f-ad43-2e6d83285be2","seconds":300}
{"level":"info","ts":"2026-11-03T15:26:54.884Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"20a8741c-e718-44a5-81e2-3936099e2fef","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:26:54.884Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"20a8741c-e718-44a5-81e2-3936099e2fef","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:26:54.884Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"20a8741c-e718-44a5-81e2-3936099e2fef","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:26:54.884Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"20a8741c-e718-44a5-81e2-3936099e2fef"}
{"level":"info","ts":"2026-11-03T15:26:54.884Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"20a8741c-e718-44a5-81e2-3936099e2fef","rayCluster":"stop-lwhxj","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:26:57.891Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"5c213fd4-bd00-4447-8594-c05e7645e33b","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:26:57.891Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"5c213fd4-bd00-4447-8594-c05e7645e33b","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:26:57.891Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"5c213fd4-bd00-4447-8594-c05e7645e33b","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:26:57.891Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"5c213fd4-bd00-4447-8594-c05e7645e33b"}
{"level":"info","ts":"2026-11-03T15:26:57.891Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"5c213fd4-bd00-4447-8594-c05e7645e33b","rayCluster":"stop-lwhxj","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:27:00.898Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"e3d4a567-0dcd-46d1-a9d4-110fa05677f3","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:27:00.898Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"e3d4a567-0dcd-46d1-a9d4-110fa05677f3","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:27:00.898Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"e3d4a567-0dcd-46d1-a9d4-110fa05677f3","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:27:00.898Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"e3d4a567-0dcd-46d1-a9d4-110fa05677f3"}
{"level":"info","ts":"2026-11-03T15:27:00.898Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"e3d4a567-0dcd-46d1-a9d4-110fa05677f3","rayCluster":"stop-lwhxj","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:27:00.904Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"e3d4a567-0dcd-46d1-a9d4-110fa05677f3","oldRayJobStatus":{"rayJobInfo":{},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","dashboardURL":"stop-lwhxj-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:26:40Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:26:30Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:27:00Z"},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","dashboardURL":"stop-lwhxj-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:26:51Z","stateTransitionTimes":{"ready":"2026-11-03T15:26:51Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:26:30Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:26:51Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:27:00.905Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"e3d4a567-0dcd-46d1-a9d4-110fa05677f3","old JobStatus":"","new JobStatus":"PENDING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-11-03T15:27:01.003Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"b2a3dc8a-8b17-414d-afe6-bfe1f88ae199","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:27:01.003Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"b2a3dc8a-8b17-414d-afe6-bfe1f88ae199","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:27:01.003Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"b2a3dc8a-8b17-414d-afe6-bfe1f88ae199","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:27:01.003Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"b2a3dc8a-8b17-414d-afe6-bfe1f88ae199"}
{"level":"info","ts":"2026-11-03T15:27:01.003Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"b2a3dc8a-8b17-414d-afe6-bfe1f88ae199","rayCluster":"stop-lwhxj","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:27:01.006Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"b2a3dc8a-8b17-414d-afe6-bfe1f88ae199","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:27:00Z"},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","dashboardURL":"stop-lwhxj-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:26:51Z","stateTransitionTimes":{"ready":"2026-11-03T15:26:51Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:26:30Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:26:51Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:27:00Z"},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","dashboardURL":"stop-lwhxj-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:26:51Z","stateTransitionTimes":{"ready":"2026-11-03T15:26:51Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:26:30Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:26:51Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:27:04.007Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"3c1d54e3-151c-43df-95fb-f85c4c2bd147","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:27:04.007Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"3c1d54e3-151c-43df-95fb-f85c4c2bd147","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:27:04.007Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"3c1d54e3-151c-43df-95fb-f85c4c2bd147","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:27:04.008Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"3c1d54e3-151c-43df-95fb-f85c4c2bd147"}
{"level":"info","ts":"2026-11-03T15:27:04.008Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"3c1d54e3-151c-43df-95fb-f85c4c2bd147","rayCluster":"stop-lwhxj","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:27:04.011Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"3c1d54e3-151c-43df-95fb-f85c4c2bd147","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:27:00Z"},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","dashboardURL":"stop-lwhxj-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:26:51Z","stateTransitionTimes":{"ready":"2026-11-03T15:26:51Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:26:30Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:26:51Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:27:00Z"},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","dashboardURL":"stop-lwhxj-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:26:51Z","stateTransitionTimes":{"ready":"2026-11-03T15:26:51Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:26:30Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:26:51Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:27:04.011Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"3c1d54e3-151c-43df-95fb-f85c4c2bd147","old JobStatus":"PENDING","new JobStatus":"RUNNING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-11-03T15:27:04.108Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"ad8440c8-269f-4fd9-866d-9df60e48f645","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:27:04.108Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"ad8440c8-269f-4fd9-866d-9df60e48f645","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:27:04.108Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"ad8440c8-269f-4fd9-866d-9df60e48f645","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:27:04.108Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"ad8440c8-269f-4fd9-866d-9df60e48f645"}
{"level":"info","ts":"2026-11-03T15:27:04.108Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"ad8440c8-269f-4fd9-866d-9df60e48f645","rayCluster":"stop-lwhxj","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:27:04.111Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"ad8440c8-269f-4fd9-866d-9df60e48f645","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:27:00Z"},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","dashboardURL":"stop-lwhxj-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:26:51Z","stateTransitionTimes":{"ready":"2026-11-03T15:26:51Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:26:30Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:26:51Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:27:00Z"},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","dashboardURL":"stop-lwhxj-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:26:51Z","stateTransitionTimes":{"ready":"2026-11-03T15:26:51Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:26:30Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:26:51Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:27:07.111Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"8cafb164-b0d5-4396-ae26-28f0dbda3b86","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:27:07.111Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"8cafb164-b0d5-4396-ae26-28f0dbda3b86","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:27:07.112Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"8cafb164-b0d5-4396-ae26-28f0dbda3b86","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:27:07.112Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"8cafb164-b0d5-4396-ae26-28f0dbda3b86"}
{"level":"info","ts":"2026-11-03T15:27:07.112Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"8cafb164-b0d5-4396-ae26-28f0dbda3b86","rayCluster":"stop-lwhxj","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:27:07.115Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"8cafb164-b0d5-4396-ae26-28f0dbda3b86","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:27:00Z"},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","dashboardURL":"stop-lwhxj-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:26:51Z","stateTransitionTimes":{"ready":"2026-11-03T15:26:51Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:26:30Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:26:51Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:27:00Z"},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","dashboardURL":"stop-lwhxj-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:26:51Z","stateTransitionTimes":{"ready":"2026-11-03T15:26:51Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:26:30Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:26:51Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:27:10.115Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"a6ff3c76-4c6c-4d00-8f9b-f01d899c2eb6","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:27:10.115Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"a6ff3c76-4c6c-4d00-8f9b-f01d899c2eb6","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:27:10.116Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"a6ff3c76-4c6c-4d00-8f9b-f01d899c2eb6","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:27:10.116Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"a6ff3c76-4c6c-4d00-8f9b-f01d899c2eb6"}
{"level":"info","ts":"2026-11-03T15:27:10.116Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"a6ff3c76-4c6c-4d00-8f9b-f01d899c2eb6","rayCluster":"stop-lwhxj","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:27:10.119Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"a6ff3c76-4c6c-4d00-8f9b-f01d899c2eb6","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:27:00Z"},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","dashboardURL":"stop-lwhxj-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:26:51Z","stateTransitionTimes":{"ready":"2026-11-03T15:26:51Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:26:30Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:26:51Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:27:00Z"},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","dashboardURL":"stop-lwhxj-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:26:51Z","stateTransitionTimes":{"ready":"2026-11-03T15:26:51Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:26:30Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:26:51Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:27:13.119Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"8c53586e-406d-4402-b599-647d865557c1","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:27:13.119Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"8c53586e-406d-4402-b599-647d865557c1","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:27:13.119Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"8c53586e-406d-4402-b599-647d865557c1","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:27:13.120Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"8c53586e-406d-4402-b599-647d865557c1"}
{"level":"info","ts":"2026-11-03T15:27:13.120Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"8c53586e-406d-4402-b599-647d865557c1","rayCluster":"stop-lwhxj","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:27:13.123Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"8c53586e-406d-4402-b599-647d865557c1","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:27:00Z"},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","dashboardURL":"stop-lwhxj-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:26:51Z","stateTransitionTimes":{"ready":"2026-11-03T15:26:51Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:26:30Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:26:51Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:27:00Z"},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","dashboardURL":"stop-lwhxj-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:26:51Z","stateTransitionTimes":{"ready":"2026-11-03T15:26:51Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:26:30Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:26:51Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:27:16.124Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"7eab6b23-b0d0-4053-bdbc-1dc8c316de9f","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:27:16.124Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"7eab6b23-b0d0-4053-bdbc-1dc8c316de9f","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:27:16.124Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"7eab6b23-b0d0-4053-bdbc-1dc8c316de9f","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:27:16.124Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"7eab6b23-b0d0-4053-bdbc-1dc8c316de9f"}
{"level":"info","ts":"2026-11-03T15:27:16.124Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"7eab6b23-b0d0-4053-bdbc-1dc8c316de9f","rayCluster":"stop-lwhxj","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:27:16.127Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"7eab6b23-b0d0-4053-bdbc-1dc8c316de9f","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:27:00Z"},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","dashboardURL":"stop-lwhxj-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:26:51Z","stateTransitionTimes":{"ready":"2026-11-03T15:26:51Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:26:30Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:26:51Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:27:00Z"},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","dashboardURL":"stop-lwhxj-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:26:51Z","stateTransitionTimes":{"ready":"2026-11-03T15:26:51Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:26:30Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:26:51Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:27:19.127Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"e3821e73-0ca9-42d4-96c2-6c05db024417","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:27:19.127Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"e3821e73-0ca9-42d4-96c2-6c05db024417","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:27:19.127Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"e3821e73-0ca9-42d4-96c2-6c05db024417","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:27:19.127Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"e3821e73-0ca9-42d4-96c2-6c05db024417"}
{"level":"info","ts":"2026-11-03T15:27:19.127Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"e3821e73-0ca9-42d4-96c2-6c05db024417","rayCluster":"stop-lwhxj","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:27:19.131Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"e3821e73-0ca9-42d4-96c2-6c05db024417","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:27:00Z"},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","dashboardURL":"stop-lwhxj-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:26:51Z","stateTransitionTimes":{"ready":"2026-11-03T15:26:51Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:26:30Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:26:51Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:27:00Z"},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","dashboardURL":"stop-lwhxj-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:26:51Z","stateTransitionTimes":{"ready":"2026-11-03T15:26:51Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:26:30Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:26:51Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:27:22.131Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"36a9c589-95a5-41b2-9b15-529eebc87836","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:27:22.131Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"36a9c589-95a5-41b2-9b15-529eebc87836","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:27:22.131Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"36a9c589-95a5-41b2-9b15-529eebc87836","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:27:22.132Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"36a9c589-95a5-41b2-9b15-529eebc87836"}
{"level":"info","ts":"2026-11-03T15:27:22.132Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"36a9c589-95a5-41b2-9b15-529eebc87836","rayCluster":"stop-lwhxj","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:27:22.135Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"36a9c589-95a5-41b2-9b15-529eebc87836","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:27:00Z"},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","dashboardURL":"stop-lwhxj-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:26:51Z","stateTransitionTimes":{"ready":"2026-11-03T15:26:51Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:26:30Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:26:51Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:27:00Z"},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","dashboardURL":"stop-lwhxj-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:26:51Z","stateTransitionTimes":{"ready":"2026-11-03T15:26:51Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:26:30Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:26:51Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:27:25.135Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"3787e68a-322c-4a3a-aa92-9300e875aac8","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:27:25.135Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"3787e68a-322c-4a3a-aa92-9300e875aac8","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:27:25.135Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"3787e68a-322c-4a3a-aa92-9300e875aac8","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:27:25.135Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"3787e68a-322c-4a3a-aa92-9300e875aac8"}
{"level":"info","ts":"2026-11-03T15:27:25.135Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"3787e68a-322c-4a3a-aa92-9300e875aac8","rayCluster":"stop-lwhxj","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:27:25.138Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"3787e68a-322c-4a3a-aa92-9300e875aac8","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:27:00Z"},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","dashboardURL":"stop-lwhxj-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:26:51Z","stateTransitionTimes":{"ready":"2026-11-03T15:26:51Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:26:30Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:26:51Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:27:00Z","endTime":"2026-11-03T15:27:22Z"},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","dashboardURL":"stop-lwhxj-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Running","message":"Job was intentionally stopped.","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:26:51Z","stateTransitionTimes":{"ready":"2026-11-03T15:26:51Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:26:30Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:26:51Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:27:25.138Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"3787e68a-322c-4a3a-aa92-9300e875aac8","old JobStatus":"RUNNING","new JobStatus":"STOPPED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-11-03T15:27:25.236Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"e6c3588f-3f0f-4723-917c-64d16a4bd8ab","JobStatus":"STOPPED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:27:25.236Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"e6c3588f-3f0f-4723-917c-64d16a4bd8ab","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:27:25.236Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"e6c3588f-3f0f-4723-917c-64d16a4bd8ab","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:27:25.236Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"e6c3588f-3f0f-4723-917c-64d16a4bd8ab"}
{"level":"info","ts":"2026-11-03T15:27:25.236Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"e6c3588f-3f0f-4723-917c-64d16a4bd8ab","rayCluster":"stop-lwhxj","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:27:25.239Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"e6c3588f-3f0f-4723-917c-64d16a4bd8ab","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:27:00Z","endTime":"2026-11-03T15:27:22Z"},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","dashboardURL":"stop-lwhxj-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Running","message":"Job was intentionally stopped.","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:26:51Z","stateTransitionTimes":{"ready":"2026-11-03T15:26:51Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:26:30Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:26:51Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:27:00Z","endTime":"2026-11-03T15:27:22Z"},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","dashboardURL":"stop-lwhxj-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Running","message":"Job was intentionally stopped.","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:26:51Z","stateTransitionTimes":{"ready":"2026-11-03T15:26:51Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:26:30Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:26:51Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:27:28.035Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"141876f8-52cc-4df2-bf7b-9a4d4c29a9c9","JobStatus":"STOPPED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:27:28.035Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"141876f8-52cc-4df2-bf7b-9a4d4c29a9c9","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:27:28.036Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"141876f8-52cc-4df2-bf7b-9a4d4c29a9c9","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:27:28.036Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"141876f8-52cc-4df2-bf7b-9a4d4c29a9c9"}
{"level":"info","ts":"2026-11-03T15:27:28.036Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"141876f8-52cc-4df2-bf7b-9a4d4c29a9c9","rayCluster":"stop-lwhxj","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:27:28.039Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"141876f8-52cc-4df2-bf7b-9a4d4c29a9c9","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:27:00Z","endTime":"2026-11-03T15:27:22Z"},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","dashboardURL":"stop-lwhxj-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Running","message":"Job was intentionally stopped.","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:26:51Z","stateTransitionTimes":{"ready":"2026-11-03T15:26:51Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:26:30Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:26:51Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:27:00Z","endTime":"2026-11-03T15:27:22Z"},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","dashboardURL":"stop-lwhxj-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Running","message":"Job was intentionally stopped.","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:26:51Z","stateTransitionTimes":{"ready":"2026-11-03T15:26:51Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:26:30Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:26:51Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:27:29.044Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"87145dca-3b21-41e3-be71-72fb6939eb51","JobStatus":"STOPPED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:27:29.044Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"87145dca-3b21-41e3-be71-72fb6939eb51","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:27:29.044Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"87145dca-3b21-41e3-be71-72fb6939eb51","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:27:29.044Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"87145dca-3b21-41e3-be71-72fb6939eb51"}
{"level":"info","ts":"2026-11-03T15:27:29.044Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"87145dca-3b21-41e3-be71-72fb6939eb51","rayCluster":"stop-lwhxj","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:27:29.048Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"87145dca-3b21-41e3-be71-72fb6939eb51","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:27:00Z","endTime":"2026-11-03T15:27:22Z"},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","dashboardURL":"stop-lwhxj-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Running","message":"Job was intentionally stopped.","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:26:51Z","stateTransitionTimes":{"ready":"2026-11-03T15:26:51Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:26:30Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:26:51Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:27:00Z","endTime":"2026-11-03T15:27:22Z"},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","dashboardURL":"stop-lwhxj-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Running","message":"Job was intentionally stopped.","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:26:51Z","stateTransitionTimes":{"ready":"2026-11-03T15:26:51Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:26:30Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:26:51Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:27:29.062Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"362a10da-98fe-4692-bc39-44e21d9ec5fb","JobStatus":"STOPPED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:27:29.062Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"362a10da-98fe-4692-bc39-44e21d9ec5fb","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:27:29.062Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"362a10da-98fe-4692-bc39-44e21d9ec5fb","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:27:29.062Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"362a10da-98fe-4692-bc39-44e21d9ec5fb"}
{"level":"info","ts":"2026-11-03T15:27:29.062Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"362a10da-98fe-4692-bc39-44e21d9ec5fb","rayCluster":"stop-lwhxj","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:27:29.065Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"362a10da-98fe-4692-bc39-44e21d9ec5fb","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:27:00Z","endTime":"2026-11-03T15:27:22Z"},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","dashboardURL":"stop-lwhxj-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Running","message":"Job was intentionally stopped.","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:26:51Z","stateTransitionTimes":{"ready":"2026-11-03T15:26:51Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:26:30Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:26:51Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:27:00Z","endTime":"2026-11-03T15:27:22Z"},"jobId":"stop-ld2x7","rayClusterName":"stop-lwhxj","dashboardURL":"stop-lwhxj-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"STOPPED","jobDeploymentStatus":"Complete","message":"Job was intentionally stopped.","startTime":"2026-11-03T15:24:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:26:51Z","stateTransitionTimes":{"ready":"2026-11-03T15:26:51Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.41","serviceIP":"10.132.0.41","podName":"stop-lwhxj-head-brfn5","serviceName":"stop-lwhxj-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:26:30Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:26:51Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:24:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:24:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:27:29.065Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"362a10da-98fe-4692-bc39-44e21d9ec5fb","old JobStatus":"STOPPED","new JobStatus":"STOPPED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Complete"}
{"level":"info","ts":"2026-11-03T15:27:29.163Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"1ff73eec-ef11-486b-9e2b-207b40679338","JobStatus":"STOPPED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:27:29.207Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"b6077aab-2347-482d-91bf-c60985e20dcc","DeletionTimestamp":"2026-11-03 15:27:29 +0000 UTC"}
{"level":"info","ts":"2026-11-03T15:27:29.207Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"b6077aab-2347-482d-91bf-c60985e20dcc","finalizer":"ray.io/rayjob-finalizer"}
{"level":"error","ts":"2026-11-03T15:27:29.271Z","logger":"controllers.RayJob","msg":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type types.RuntimeEnvType","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-kbx4w"},"reconcileID":"1be833e0-815f-4e0b-acd4-c1bac3fb8032","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type types.RuntimeEnvType","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.validateRayJob\n\t/workspace/controllers/ray/rayjob_controller.go:526\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:145\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:27:29.271Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-kbx4w"},"reconcileID":"1be833e0-815f-4e0b-acd4-c1bac3fb8032","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobDeploymentStatus":"ValidationFailed","reason":"ValidationFailed","message":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type types.RuntimeEnvType","rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:27:29.271Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-kbx4w"},"reconcileID":"1be833e0-815f-4e0b-acd4-c1bac3fb8032","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"ValidationFailed"}
{"level":"info","ts":"2026-11-03T15:27:29.295Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"reconcileID":"2ec97ba5-82fe-4e4a-b6c2-94e5f45b38d4"}
{"level":"info","ts":"2026-11-03T15:27:29.295Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"stop-lwhxj","reconcileID":"70e8a459-3272-4573-a641-c487cab09291","namespacedName":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:27:29.295Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"stop-lwhxj","reconcileID":"70e8a459-3272-4573-a641-c487cab09291","cluster":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:27:29.295Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"stop-lwhxj","reconcileID":"70e8a459-3272-4573-a641-c487cab09291","cluster":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:27:29.295Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"stop-lwhxj","reconcileID":"70e8a459-3272-4573-a641-c487cab09291","namespace":"test-ns-kbx4w","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-11-03T15:27:29.329Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"2da7dcd3-d248-4882-b33c-cb4ca3d749d3"}
{"level":"error","ts":"2026-11-03T15:27:29.329Z","logger":"controllers.RayJob","msg":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type types.RuntimeEnvType","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-kbx4w"},"reconcileID":"0b7ce005-e846-4348-9230-b21391072891","error":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type types.RuntimeEnvType","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.validateRayJob\n\t/workspace/controllers/ray/rayjob_controller.go:526\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:145\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:27:29.330Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-kbx4w"},"reconcileID":"0b7ce005-e846-4348-9230-b21391072891","oldRayJobStatus":{"rayJobInfo":{},"jobDeploymentStatus":"ValidationFailed","reason":"ValidationFailed","message":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type types.RuntimeEnvType","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobDeploymentStatus":"ValidationFailed","reason":"ValidationFailed","message":"failed to unmarshal RuntimeEnvYAML: invalid_yaml_string: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type types.RuntimeEnvType","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:27:29.348Z","logger":"authentication-controller","msg":"Deleted shared ReferenceGrant (last cluster with auth)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"stop-lwhxj","reconcileID":"70e8a459-3272-4573-a641-c487cab09291","name":"kuberay-gateway-access","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:27:29.401Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"stop-lwhxj","reconcileID":"70e8a459-3272-4573-a641-c487cab09291","name":"test-ns-kbx4w-stop-lwhxj","namespace":"opendatahub"}
{"level":"info","ts":"2026-11-03T15:27:29.453Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"stop-lwhxj","reconcileID":"70e8a459-3272-4573-a641-c487cab09291","configMap":"kube-rbac-proxy-config-stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:27:29.506Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"stop-lwhxj","reconcileID":"70e8a459-3272-4573-a641-c487cab09291","serviceAccount":"stop-lwhxj-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-11-03T15:27:29.689Z","logger":"raycluster-resource","msg":"default","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:27:29.689Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"stop-lwhxj","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:27:29.868Z","logger":"raycluster-resource","msg":"validate update","name":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:27:29.923Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"stop-lwhxj","reconcileID":"70e8a459-3272-4573-a641-c487cab09291","cluster":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:27:29.923Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"stop-lwhxj","reconcileID":"cf954ef7-4fcf-4990-8b27-e27d7fa1af4c","namespacedName":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:27:29.923Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"stop-lwhxj","reconcileID":"cf954ef7-4fcf-4990-8b27-e27d7fa1af4c","cluster":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:27:29.923Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"stop-lwhxj","reconcileID":"cf954ef7-4fcf-4990-8b27-e27d7fa1af4c","cluster":"stop-lwhxj"}
{"level":"info","ts":"2026-11-03T15:27:29.923Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"stop-lwhxj","reconcileID":"cf954ef7-4fcf-4990-8b27-e27d7fa1af4c","namespace":"test-ns-kbx4w","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-11-03T15:27:29.923Z","logger":"authentication-controller","msg":"ReferenceGrant already deleted","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"stop-lwhxj","reconcileID":"cf954ef7-4fcf-4990-8b27-e27d7fa1af4c","name":"kuberay-gateway-access"}
{"level":"info","ts":"2026-11-03T15:27:29.924Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"stop","namespace":"test-ns-kbx4w"},"reconcileID":"ffd5cbb0-d451-4622-96eb-f13c55a8350a"}
{"level":"error","ts":"2026-11-03T15:27:29.986Z","logger":"authentication-controller","msg":"Failed to remove finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"stop-lwhxj","reconcileID":"cf954ef7-4fcf-4990-8b27-e27d7fa1af4c","error":"rayclusters.ray.io \"stop-lwhxj\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleDeletion\n\t/workspace/controllers/ray/authentication_controller.go:195\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:27:29.986Z","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"stop-lwhxj","reconcileID":"cf954ef7-4fcf-4990-8b27-e27d7fa1af4c"}
{"level":"error","ts":"2026-11-03T15:27:29.986Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"stop-lwhxj","reconcileID":"cf954ef7-4fcf-4990-8b27-e27d7fa1af4c","error":"rayclusters.ray.io \"stop-lwhxj\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:27:29.986Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"stop-lwhxj","reconcileID":"6384c497-c27b-4836-980d-9f70340e7266","namespacedName":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:27:29.986Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"stop-lwhxj","reconcileID":"6384c497-c27b-4836-980d-9f70340e7266"}
{"level":"info","ts":"2026-11-03T15:27:29.986Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"stop-lwhxj","reconcileID":"6384c497-c27b-4836-980d-9f70340e7266","namespace":"test-ns-kbx4w","authClusters":0}
{"level":"info","ts":"2026-11-03T15:27:29.986Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"stop-lwhxj","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"stop-lwhxj","reconcileID":"6384c497-c27b-4836-980d-9f70340e7266","namespace":"test-ns-kbx4w"}
{"level":"error","ts":"2026-11-03T15:27:30.286Z","logger":"controllers.RayJob","msg":"The RayJob metadata is invalid: RayJob name should be no more than 47 characters","RayJob":{"name":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","namespace":"test-ns-kbx4w"},"reconcileID":"be664e98-985a-4b85-b4a2-1c27fcb50959","error":"The RayJob metadata is invalid: RayJob name should be no more than 47 characters","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.validateRayJob\n\t/workspace/controllers/ray/rayjob_controller.go:526\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:145\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:27:30.286Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","namespace":"test-ns-kbx4w"},"reconcileID":"be664e98-985a-4b85-b4a2-1c27fcb50959","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobDeploymentStatus":"ValidationFailed","reason":"ValidationFailed","message":"The RayJob metadata is invalid: RayJob name should be no more than 47 characters","rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:27:30.286Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","namespace":"test-ns-kbx4w"},"reconcileID":"be664e98-985a-4b85-b4a2-1c27fcb50959","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"ValidationFailed"}
{"level":"error","ts":"2026-11-03T15:27:30.345Z","logger":"controllers.RayJob","msg":"The RayJob metadata is invalid: RayJob name should be no more than 47 characters","RayJob":{"name":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","namespace":"test-ns-kbx4w"},"reconcileID":"88ebe612-ac79-4c7a-98fb-618cd2877281","error":"The RayJob metadata is invalid: RayJob name should be no more than 47 characters","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.validateRayJob\n\t/workspace/controllers/ray/rayjob_controller.go:526\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:145\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:27:30.345Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","namespace":"test-ns-kbx4w"},"reconcileID":"88ebe612-ac79-4c7a-98fb-618cd2877281","oldRayJobStatus":{"rayJobInfo":{},"jobDeploymentStatus":"ValidationFailed","reason":"ValidationFailed","message":"The RayJob metadata is invalid: RayJob name should be no more than 47 characters","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobDeploymentStatus":"ValidationFailed","reason":"ValidationFailed","message":"The RayJob metadata is invalid: RayJob name should be no more than 47 characters","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:27:31.347Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"9151f90f-307c-4649-abc6-fbc37b0b8120","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:27:31.347Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"9151f90f-307c-4649-abc6-fbc37b0b8120","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-11-03T15:27:31.407Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"9151f90f-307c-4649-abc6-fbc37b0b8120"}
{"level":"info","ts":"2026-11-03T15:27:31.407Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"9151f90f-307c-4649-abc6-fbc37b0b8120","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-11-03T15:27:31.407Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"9151f90f-307c-4649-abc6-fbc37b0b8120","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-cdx2h","rayClusterName":"long-running-hvml9","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:31Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:27:31.407Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"9151f90f-307c-4649-abc6-fbc37b0b8120","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:27:31.466Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"8f04858b-0e01-4885-aa38-28df711647d9","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:27:31.466Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"8f04858b-0e01-4885-aa38-28df711647d9","name":"long-running-hvml9"}
{"level":"info","ts":"2026-11-03T15:27:31.466Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"8f04858b-0e01-4885-aa38-28df711647d9","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:27:31.466Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"8f04858b-0e01-4885-aa38-28df711647d9","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:27:31.522Z","logger":"raycluster-resource","msg":"default","name":"long-running-hvml9"}
{"level":"info","ts":"2026-11-03T15:27:31.522Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"long-running-hvml9","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:27:31.587Z","logger":"raycluster-resource","msg":"validate create","name":"long-running-hvml9"}
{"level":"info","ts":"2026-11-03T15:27:31.641Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"8f04858b-0e01-4885-aa38-28df711647d9","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:27:31.641Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"8f04858b-0e01-4885-aa38-28df711647d9"}
{"level":"info","ts":"2026-11-03T15:27:31.641Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"8f04858b-0e01-4885-aa38-28df711647d9","RayCluster":"long-running-hvml9","State":""}
{"level":"info","ts":"2026-11-03T15:27:31.641Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"8f04858b-0e01-4885-aa38-28df711647d9","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-cdx2h","rayClusterName":"long-running-hvml9","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:31Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-cdx2h","rayClusterName":"long-running-hvml9","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:31Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:27:31.641Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"ee6b51f1-31d6-4753-93e8-ac0433701ceb","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:27:31.641Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"ee6b51f1-31d6-4753-93e8-ac0433701ceb","name":"long-running-hvml9"}
{"level":"info","ts":"2026-11-03T15:27:31.641Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"long-running-hvml9","reconcileID":"25b708b0-7c44-4efc-949e-6615cf19c1ba","namespacedName":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:27:31.641Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"ee6b51f1-31d6-4753-93e8-ac0433701ceb","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:27:31.641Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"long-running-hvml9","reconcileID":"25b708b0-7c44-4efc-949e-6615cf19c1ba","mode":"IntegratedOAuth","cluster":"long-running-hvml9"}
{"level":"info","ts":"2026-11-03T15:27:31.641Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"long-running-hvml9","reconcileID":"25b708b0-7c44-4efc-949e-6615cf19c1ba","cluster":"long-running-hvml9"}
{"level":"info","ts":"2026-11-03T15:27:31.642Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"long-running-hvml9","reconcileID":"25b708b0-7c44-4efc-949e-6615cf19c1ba","cluster":"long-running-hvml9"}
{"level":"info","ts":"2026-11-03T15:27:31.642Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"ee6b51f1-31d6-4753-93e8-ac0433701ceb"}
{"level":"info","ts":"2026-11-03T15:27:31.642Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"ee6b51f1-31d6-4753-93e8-ac0433701ceb","RayCluster":"long-running-hvml9","State":""}
{"level":"info","ts":"2026-11-03T15:27:31.642Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"ee6b51f1-31d6-4753-93e8-ac0433701ceb","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-cdx2h","rayClusterName":"long-running-hvml9","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:31Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-cdx2h","rayClusterName":"long-running-hvml9","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:31Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:27:31.642Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"d152cfbe-f2f0-41ef-973e-7c874a46949a"}
{"level":"info","ts":"2026-11-03T15:27:31.642Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"d152cfbe-f2f0-41ef-973e-7c874a46949a"}
{"level":"info","ts":"2026-11-03T15:27:31.642Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"ed15ba42-10c9-4069-9053-38a56bc11cb5","cluster":"long-running-hvml9"}
{"level":"info","ts":"2026-11-03T15:27:31.642Z","logger":"controllers.NetworkPolicy","msg":"Reconciling NetworkPolicies for RayCluster","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"178c4688-f2bc-43fe-a37f-c5f85582b27a","cluster":"long-running-hvml9","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:27:31.642Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"ab51f0aa-2b6d-4f9c-b8aa-eff5380f3cbb","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:27:31.642Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"ab51f0aa-2b6d-4f9c-b8aa-eff5380f3cbb","name":"long-running-hvml9"}
{"level":"info","ts":"2026-11-03T15:27:31.642Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"ab51f0aa-2b6d-4f9c-b8aa-eff5380f3cbb","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:27:31.642Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"ab51f0aa-2b6d-4f9c-b8aa-eff5380f3cbb"}
{"level":"info","ts":"2026-11-03T15:27:31.642Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"ab51f0aa-2b6d-4f9c-b8aa-eff5380f3cbb","RayCluster":"long-running-hvml9","State":""}
{"level":"info","ts":"2026-11-03T15:27:31.642Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"ab51f0aa-2b6d-4f9c-b8aa-eff5380f3cbb","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-cdx2h","rayClusterName":"long-running-hvml9","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:31Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-cdx2h","rayClusterName":"long-running-hvml9","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:31Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:27:31.696Z","logger":"controllers.NetworkPolicy","msg":"Successfully created NetworkPolicy","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"178c4688-f2bc-43fe-a37f-c5f85582b27a","name":"long-running-hvml9-head"}
{"level":"info","ts":"2026-11-03T15:27:31.697Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"d152cfbe-f2f0-41ef-973e-7c874a46949a","name":"long-running-hvml9-head-svc"}
{"level":"info","ts":"2026-11-03T15:27:31.697Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"d152cfbe-f2f0-41ef-973e-7c874a46949a"}
{"level":"info","ts":"2026-11-03T15:27:31.700Z","logger":"raycluster-resource","msg":"default","name":"long-running-hvml9"}
{"level":"info","ts":"2026-11-03T15:27:31.700Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"long-running-hvml9","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:27:31.746Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"d152cfbe-f2f0-41ef-973e-7c874a46949a","error":"certificate ray-head-cert-long-running-hvml9 not found: certificates.cert-manager.io \"ray-head-cert-long-running-hvml9\" not found"}
{"level":"error","ts":"2026-11-03T15:27:31.746Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"d152cfbe-f2f0-41ef-973e-7c874a46949a","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-hvml9 not found: certificates.cert-manager.io \"ray-head-cert-long-running-hvml9\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:27:31.747Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"d152cfbe-f2f0-41ef-973e-7c874a46949a","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-hvml9 not found: certificates.cert-manager.io \"ray-head-cert-long-running-hvml9\" not found"}
{"level":"info","ts":"2026-11-03T15:27:31.747Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"d152cfbe-f2f0-41ef-973e-7c874a46949a","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"long-running-hvml9","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:27:31.747Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"d152cfbe-f2f0-41ef-973e-7c874a46949a","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"long-running-hvml9","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:27:31.747Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"d152cfbe-f2f0-41ef-973e-7c874a46949a","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"long-running-hvml9","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:27:31.747Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"d152cfbe-f2f0-41ef-973e-7c874a46949a","name":"long-running-hvml9","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-hvml9-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:27:31.750Z","logger":"controllers.NetworkPolicy","msg":"Successfully created NetworkPolicy","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"178c4688-f2bc-43fe-a37f-c5f85582b27a","name":"long-running-hvml9-workers-small-group"}
{"level":"info","ts":"2026-11-03T15:27:31.750Z","logger":"controllers.NetworkPolicy","msg":"Successfully reconciled NetworkPolicies for RayCluster","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"178c4688-f2bc-43fe-a37f-c5f85582b27a","cluster":"long-running-hvml9","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:27:31.750Z","logger":"controllers.NetworkPolicy","msg":"Reconciling NetworkPolicies for RayCluster","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"8bc9d1ad-ff72-4a5a-9b24-67a18d6f35ba","cluster":"long-running-hvml9","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:27:31.760Z","logger":"raycluster-resource","msg":"validate update","name":"long-running-hvml9"}
{"level":"info","ts":"2026-11-03T15:27:31.806Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"d152cfbe-f2f0-41ef-973e-7c874a46949a"}
{"level":"error","ts":"2026-11-03T15:27:31.806Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"d152cfbe-f2f0-41ef-973e-7c874a46949a","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-hvml9 not found: certificates.cert-manager.io \"ray-head-cert-long-running-hvml9\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:27:31.806Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"de6b4ed7-f164-4f8f-907d-95c768182982"}
{"level":"info","ts":"2026-11-03T15:27:31.806Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"de6b4ed7-f164-4f8f-907d-95c768182982"}
{"level":"info","ts":"2026-11-03T15:27:31.807Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"de6b4ed7-f164-4f8f-907d-95c768182982","1 head service found":"long-running-hvml9-head-svc"}
{"level":"info","ts":"2026-11-03T15:27:31.807Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"de6b4ed7-f164-4f8f-907d-95c768182982"}
{"level":"info","ts":"2026-11-03T15:27:31.807Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"676359e9-55a4-460e-a8c3-14c7db120684","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:27:31.807Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"676359e9-55a4-460e-a8c3-14c7db120684","name":"long-running-hvml9"}
{"level":"info","ts":"2026-11-03T15:27:31.807Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"676359e9-55a4-460e-a8c3-14c7db120684","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:27:31.807Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"676359e9-55a4-460e-a8c3-14c7db120684"}
{"level":"info","ts":"2026-11-03T15:27:31.807Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"676359e9-55a4-460e-a8c3-14c7db120684","RayCluster":"long-running-hvml9","State":""}
{"level":"info","ts":"2026-11-03T15:27:31.807Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"676359e9-55a4-460e-a8c3-14c7db120684","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-cdx2h","rayClusterName":"long-running-hvml9","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:31Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-cdx2h","rayClusterName":"long-running-hvml9","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:31Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-hvml9-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:27:31.807Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"676359e9-55a4-460e-a8c3-14c7db120684","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:27:31.821Z","logger":"raycluster-resource","msg":"default","name":"long-running-hvml9"}
{"level":"info","ts":"2026-11-03T15:27:31.821Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"long-running-hvml9","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:27:31.856Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"de6b4ed7-f164-4f8f-907d-95c768182982","error":"certificate ray-head-cert-long-running-hvml9 not found: certificates.cert-manager.io \"ray-head-cert-long-running-hvml9\" not found"}
{"level":"error","ts":"2026-11-03T15:27:31.856Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"de6b4ed7-f164-4f8f-907d-95c768182982","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-hvml9 not found: certificates.cert-manager.io \"ray-head-cert-long-running-hvml9\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:27:31.856Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"de6b4ed7-f164-4f8f-907d-95c768182982","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-hvml9 not found: certificates.cert-manager.io \"ray-head-cert-long-running-hvml9\" not found"}
{"level":"info","ts":"2026-11-03T15:27:31.856Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"de6b4ed7-f164-4f8f-907d-95c768182982","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"long-running-hvml9","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:27:31.856Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"de6b4ed7-f164-4f8f-907d-95c768182982","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"long-running-hvml9","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:27:31.856Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"de6b4ed7-f164-4f8f-907d-95c768182982","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"long-running-hvml9","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:27:31.856Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"de6b4ed7-f164-4f8f-907d-95c768182982","name":"long-running-hvml9","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-hvml9-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:27:31.861Z","logger":"controllers.NetworkPolicy","msg":"Successfully reconciled NetworkPolicies for RayCluster","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"8bc9d1ad-ff72-4a5a-9b24-67a18d6f35ba","cluster":"long-running-hvml9","namespace":"test-ns-kbx4w"}
{"level":"error","ts":"2026-11-03T15:27:31.870Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"long-running-hvml9","reconcileID":"25b708b0-7c44-4efc-949e-6615cf19c1ba","error":"failed to add finalizer: Operation cannot be fulfilled on rayclusters.ray.io \"long-running-hvml9\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:167\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:27:31.870Z","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"long-running-hvml9","reconcileID":"25b708b0-7c44-4efc-949e-6615cf19c1ba"}
{"level":"error","ts":"2026-11-03T15:27:31.870Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"long-running-hvml9","reconcileID":"25b708b0-7c44-4efc-949e-6615cf19c1ba","error":"failed to add finalizer: Operation cannot be fulfilled on rayclusters.ray.io \"long-running-hvml9\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:27:31.870Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"long-running-hvml9","reconcileID":"0e1de1bd-84ba-4027-b7fb-64106be19e7d","namespacedName":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:27:31.870Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"long-running-hvml9","reconcileID":"0e1de1bd-84ba-4027-b7fb-64106be19e7d","mode":"IntegratedOAuth","cluster":"long-running-hvml9"}
{"level":"info","ts":"2026-11-03T15:27:31.870Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"long-running-hvml9","reconcileID":"0e1de1bd-84ba-4027-b7fb-64106be19e7d","cluster":"long-running-hvml9"}
{"level":"info","ts":"2026-11-03T15:27:31.870Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"long-running-hvml9","reconcileID":"0e1de1bd-84ba-4027-b7fb-64106be19e7d","cluster":"long-running-hvml9"}
{"level":"info","ts":"2026-11-03T15:27:31.871Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"1f0de35c-79d0-4270-8a61-197001552849","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:27:31.871Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"1f0de35c-79d0-4270-8a61-197001552849","name":"long-running-hvml9"}
{"level":"info","ts":"2026-11-03T15:27:31.871Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"1f0de35c-79d0-4270-8a61-197001552849","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:27:31.871Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"1f0de35c-79d0-4270-8a61-197001552849"}
{"level":"info","ts":"2026-11-03T15:27:31.871Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"1f0de35c-79d0-4270-8a61-197001552849","RayCluster":"long-running-hvml9","State":""}
{"level":"info","ts":"2026-11-03T15:27:31.871Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"1f0de35c-79d0-4270-8a61-197001552849","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-cdx2h","rayClusterName":"long-running-hvml9","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:31Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-hvml9-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-cdx2h","rayClusterName":"long-running-hvml9","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:31Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-hvml9-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:27:31.911Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"de6b4ed7-f164-4f8f-907d-95c768182982","name":"long-running-hvml9","error":"Operation cannot be fulfilled on rayclusters.ray.io \"long-running-hvml9\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-kbx4w","name":"long-running-hvml9"}}
{"level":"info","ts":"2026-11-03T15:27:31.911Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"de6b4ed7-f164-4f8f-907d-95c768182982"}
{"level":"error","ts":"2026-11-03T15:27:31.911Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"de6b4ed7-f164-4f8f-907d-95c768182982","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-hvml9 not found: certificates.cert-manager.io \"ray-head-cert-long-running-hvml9\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:27:31.922Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"60aeb5a5-c3fb-4f9f-a711-07dc30f9b6e8"}
{"level":"info","ts":"2026-11-03T15:27:31.922Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"60aeb5a5-c3fb-4f9f-a711-07dc30f9b6e8"}
{"level":"info","ts":"2026-11-03T15:27:31.922Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"60aeb5a5-c3fb-4f9f-a711-07dc30f9b6e8","1 head service found":"long-running-hvml9-head-svc"}
{"level":"info","ts":"2026-11-03T15:27:31.922Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"60aeb5a5-c3fb-4f9f-a711-07dc30f9b6e8"}
{"level":"info","ts":"2026-11-03T15:27:31.929Z","logger":"raycluster-resource","msg":"default","name":"long-running-hvml9"}
{"level":"info","ts":"2026-11-03T15:27:31.929Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"long-running-hvml9","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:27:31.972Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"60aeb5a5-c3fb-4f9f-a711-07dc30f9b6e8","error":"certificate ray-head-cert-long-running-hvml9 not found: certificates.cert-manager.io \"ray-head-cert-long-running-hvml9\" not found"}
{"level":"error","ts":"2026-11-03T15:27:31.972Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"60aeb5a5-c3fb-4f9f-a711-07dc30f9b6e8","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-hvml9 not found: certificates.cert-manager.io \"ray-head-cert-long-running-hvml9\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:27:31.972Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"60aeb5a5-c3fb-4f9f-a711-07dc30f9b6e8","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-hvml9 not found: certificates.cert-manager.io \"ray-head-cert-long-running-hvml9\" not found"}
{"level":"info","ts":"2026-11-03T15:27:31.972Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"60aeb5a5-c3fb-4f9f-a711-07dc30f9b6e8","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"long-running-hvml9","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:27:31.972Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"60aeb5a5-c3fb-4f9f-a711-07dc30f9b6e8","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"long-running-hvml9","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:27:31.972Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"60aeb5a5-c3fb-4f9f-a711-07dc30f9b6e8","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"long-running-hvml9","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:27:31.972Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"60aeb5a5-c3fb-4f9f-a711-07dc30f9b6e8"}
{"level":"error","ts":"2026-11-03T15:27:31.972Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"60aeb5a5-c3fb-4f9f-a711-07dc30f9b6e8","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-hvml9 not found: certificates.cert-manager.io \"ray-head-cert-long-running-hvml9\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:27:31.987Z","logger":"raycluster-resource","msg":"validate update","name":"long-running-hvml9"}
{"level":"info","ts":"2026-11-03T15:27:31.993Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"775d40f3-3281-45ba-81e6-318042295d7a"}
{"level":"info","ts":"2026-11-03T15:27:31.993Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"775d40f3-3281-45ba-81e6-318042295d7a"}
{"level":"info","ts":"2026-11-03T15:27:31.993Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"775d40f3-3281-45ba-81e6-318042295d7a","1 head service found":"long-running-hvml9-head-svc"}
{"level":"info","ts":"2026-11-03T15:27:31.993Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"775d40f3-3281-45ba-81e6-318042295d7a"}
{"level":"info","ts":"2026-11-03T15:27:32.042Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"775d40f3-3281-45ba-81e6-318042295d7a","error":"certificate ray-head-cert-long-running-hvml9 not found: certificates.cert-manager.io \"ray-head-cert-long-running-hvml9\" not found"}
{"level":"error","ts":"2026-11-03T15:27:32.042Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"775d40f3-3281-45ba-81e6-318042295d7a","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-hvml9 not found: certificates.cert-manager.io \"ray-head-cert-long-running-hvml9\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:27:32.042Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"775d40f3-3281-45ba-81e6-318042295d7a","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-hvml9 not found: certificates.cert-manager.io \"ray-head-cert-long-running-hvml9\" not found"}
{"level":"info","ts":"2026-11-03T15:27:32.042Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"775d40f3-3281-45ba-81e6-318042295d7a","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"long-running-hvml9","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:27:32.042Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"775d40f3-3281-45ba-81e6-318042295d7a","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"long-running-hvml9","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:27:32.042Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"775d40f3-3281-45ba-81e6-318042295d7a","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"long-running-hvml9","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:27:32.042Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"775d40f3-3281-45ba-81e6-318042295d7a"}
{"level":"error","ts":"2026-11-03T15:27:32.042Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"775d40f3-3281-45ba-81e6-318042295d7a","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-hvml9 not found: certificates.cert-manager.io \"ray-head-cert-long-running-hvml9\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:27:32.044Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"c349688e-36b1-4978-8bf7-7f83c1182773","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:27:32.044Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"c349688e-36b1-4978-8bf7-7f83c1182773","name":"long-running-hvml9"}
{"level":"info","ts":"2026-11-03T15:27:32.044Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"c349688e-36b1-4978-8bf7-7f83c1182773","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:27:32.044Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"c349688e-36b1-4978-8bf7-7f83c1182773"}
{"level":"info","ts":"2026-11-03T15:27:32.044Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"c349688e-36b1-4978-8bf7-7f83c1182773","RayCluster":"long-running-hvml9","State":""}
{"level":"info","ts":"2026-11-03T15:27:32.044Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"c349688e-36b1-4978-8bf7-7f83c1182773","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-cdx2h","rayClusterName":"long-running-hvml9","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:31Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-hvml9-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-cdx2h","rayClusterName":"long-running-hvml9","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:31Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-hvml9-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:27:32.083Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"6043b9f4-871c-45a0-9b91-f3c150162874"}
{"level":"info","ts":"2026-11-03T15:27:32.083Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"6043b9f4-871c-45a0-9b91-f3c150162874"}
{"level":"info","ts":"2026-11-03T15:27:32.083Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"6043b9f4-871c-45a0-9b91-f3c150162874","1 head service found":"long-running-hvml9-head-svc"}
{"level":"info","ts":"2026-11-03T15:27:32.083Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"6043b9f4-871c-45a0-9b91-f3c150162874"}
{"level":"info","ts":"2026-11-03T15:27:32.095Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"long-running-hvml9","reconcileID":"0e1de1bd-84ba-4027-b7fb-64106be19e7d","name":"long-running-hvml9-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-11-03T15:27:32.133Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"6043b9f4-871c-45a0-9b91-f3c150162874","error":"certificate ray-head-cert-long-running-hvml9 not found: certificates.cert-manager.io \"ray-head-cert-long-running-hvml9\" not found"}
{"level":"error","ts":"2026-11-03T15:27:32.133Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"6043b9f4-871c-45a0-9b91-f3c150162874","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-hvml9 not found: certificates.cert-manager.io \"ray-head-cert-long-running-hvml9\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:27:32.133Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"6043b9f4-871c-45a0-9b91-f3c150162874","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-hvml9 not found: certificates.cert-manager.io \"ray-head-cert-long-running-hvml9\" not found"}
{"level":"info","ts":"2026-11-03T15:27:32.133Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"6043b9f4-871c-45a0-9b91-f3c150162874","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"long-running-hvml9","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:27:32.133Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"6043b9f4-871c-45a0-9b91-f3c150162874","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"long-running-hvml9","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:27:32.133Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"6043b9f4-871c-45a0-9b91-f3c150162874","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"long-running-hvml9","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:27:32.133Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"6043b9f4-871c-45a0-9b91-f3c150162874"}
{"level":"error","ts":"2026-11-03T15:27:32.133Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"6043b9f4-871c-45a0-9b91-f3c150162874","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-hvml9 not found: certificates.cert-manager.io \"ray-head-cert-long-running-hvml9\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:27:32.147Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"long-running-hvml9","reconcileID":"0e1de1bd-84ba-4027-b7fb-64106be19e7d","name":"kuberay-gateway-access","namespace":"test-ns-kbx4w","operation":"created"}
{"level":"info","ts":"2026-11-03T15:27:32.208Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"long-running-hvml9","reconcileID":"0e1de1bd-84ba-4027-b7fb-64106be19e7d","name":"test-ns-kbx4w-long-running-hvml9","operation":"created"}
{"level":"info","ts":"2026-11-03T15:27:32.213Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"c42c5e91-4734-4f09-a234-00e54b6f8c4b"}
{"level":"info","ts":"2026-11-03T15:27:32.213Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"c42c5e91-4734-4f09-a234-00e54b6f8c4b"}
{"level":"info","ts":"2026-11-03T15:27:32.213Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"c42c5e91-4734-4f09-a234-00e54b6f8c4b","1 head service found":"long-running-hvml9-head-svc"}
{"level":"info","ts":"2026-11-03T15:27:32.213Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"c42c5e91-4734-4f09-a234-00e54b6f8c4b"}
{"level":"info","ts":"2026-11-03T15:27:32.260Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"long-running-hvml9","reconcileID":"0e1de1bd-84ba-4027-b7fb-64106be19e7d","name":"kube-rbac-proxy-config-long-running-hvml9","operation":"created"}
{"level":"info","ts":"2026-11-03T15:27:32.265Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"c42c5e91-4734-4f09-a234-00e54b6f8c4b","error":"certificate ray-head-cert-long-running-hvml9 not found: certificates.cert-manager.io \"ray-head-cert-long-running-hvml9\" not found"}
{"level":"error","ts":"2026-11-03T15:27:32.265Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"c42c5e91-4734-4f09-a234-00e54b6f8c4b","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-hvml9 not found: certificates.cert-manager.io \"ray-head-cert-long-running-hvml9\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:27:32.265Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"c42c5e91-4734-4f09-a234-00e54b6f8c4b","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-hvml9 not found: certificates.cert-manager.io \"ray-head-cert-long-running-hvml9\" not found"}
{"level":"info","ts":"2026-11-03T15:27:32.265Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"c42c5e91-4734-4f09-a234-00e54b6f8c4b","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"long-running-hvml9","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:27:32.265Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"c42c5e91-4734-4f09-a234-00e54b6f8c4b","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"long-running-hvml9","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:27:32.265Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"c42c5e91-4734-4f09-a234-00e54b6f8c4b","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"long-running-hvml9","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:27:32.265Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"c42c5e91-4734-4f09-a234-00e54b6f8c4b"}
{"level":"error","ts":"2026-11-03T15:27:32.265Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"c42c5e91-4734-4f09-a234-00e54b6f8c4b","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-hvml9 not found: certificates.cert-manager.io \"ray-head-cert-long-running-hvml9\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-03T15:27:32.315Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"long-running-hvml9","reconcileID":"0e1de1bd-84ba-4027-b7fb-64106be19e7d","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"long-running-hvml9\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:167\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:27:32.315Z","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"long-running-hvml9","reconcileID":"0e1de1bd-84ba-4027-b7fb-64106be19e7d"}
{"level":"error","ts":"2026-11-03T15:27:32.315Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"long-running-hvml9","reconcileID":"0e1de1bd-84ba-4027-b7fb-64106be19e7d","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"long-running-hvml9\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:27:32.315Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"long-running-hvml9","reconcileID":"21cf7be2-729a-4ab5-ae78-44042cd771e2","namespacedName":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:27:32.315Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"long-running-hvml9","reconcileID":"21cf7be2-729a-4ab5-ae78-44042cd771e2","mode":"IntegratedOAuth","cluster":"long-running-hvml9"}
{"level":"info","ts":"2026-11-03T15:27:32.315Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"long-running-hvml9","reconcileID":"21cf7be2-729a-4ab5-ae78-44042cd771e2","cluster":"long-running-hvml9"}
{"level":"info","ts":"2026-11-03T15:27:32.380Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"long-running-hvml9","reconcileID":"21cf7be2-729a-4ab5-ae78-44042cd771e2","cluster":"long-running-hvml9"}
{"level":"info","ts":"2026-11-03T15:27:32.380Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"3dc3b29f-ef6d-4bed-b6eb-e34c9a2bf319","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:27:32.380Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"3dc3b29f-ef6d-4bed-b6eb-e34c9a2bf319","name":"long-running-hvml9"}
{"level":"info","ts":"2026-11-03T15:27:32.380Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"3dc3b29f-ef6d-4bed-b6eb-e34c9a2bf319","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:27:32.380Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"3dc3b29f-ef6d-4bed-b6eb-e34c9a2bf319"}
{"level":"info","ts":"2026-11-03T15:27:32.380Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"3dc3b29f-ef6d-4bed-b6eb-e34c9a2bf319","RayCluster":"long-running-hvml9","State":""}
{"level":"info","ts":"2026-11-03T15:27:32.380Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"3dc3b29f-ef6d-4bed-b6eb-e34c9a2bf319","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-cdx2h","rayClusterName":"long-running-hvml9","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:31Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-hvml9-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-cdx2h","rayClusterName":"long-running-hvml9","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:31Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-hvml9-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:32Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:27:32.380Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"3dc3b29f-ef6d-4bed-b6eb-e34c9a2bf319","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:27:32.426Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"91f1eec3-a9c6-4908-a09b-a7165cb8b5d4"}
{"level":"info","ts":"2026-11-03T15:27:32.426Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"91f1eec3-a9c6-4908-a09b-a7165cb8b5d4"}
{"level":"info","ts":"2026-11-03T15:27:32.426Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"91f1eec3-a9c6-4908-a09b-a7165cb8b5d4","1 head service found":"long-running-hvml9-head-svc"}
{"level":"info","ts":"2026-11-03T15:27:32.426Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"91f1eec3-a9c6-4908-a09b-a7165cb8b5d4"}
{"level":"info","ts":"2026-11-03T15:27:32.441Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"d92bea38-f2b7-4c4f-9711-dbe603907127","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:27:32.441Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"d92bea38-f2b7-4c4f-9711-dbe603907127","name":"long-running-hvml9"}
{"level":"info","ts":"2026-11-03T15:27:32.441Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"d92bea38-f2b7-4c4f-9711-dbe603907127","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:27:32.441Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"d92bea38-f2b7-4c4f-9711-dbe603907127"}
{"level":"info","ts":"2026-11-03T15:27:32.441Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"d92bea38-f2b7-4c4f-9711-dbe603907127","RayCluster":"long-running-hvml9","State":""}
{"level":"info","ts":"2026-11-03T15:27:32.441Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"d92bea38-f2b7-4c4f-9711-dbe603907127","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-cdx2h","rayClusterName":"long-running-hvml9","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:31Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-hvml9-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:32Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-cdx2h","rayClusterName":"long-running-hvml9","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:31Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-hvml9-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:32Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:27:32.476Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"91f1eec3-a9c6-4908-a09b-a7165cb8b5d4","error":"certificate ray-head-cert-long-running-hvml9 is not ready at current generation 1"}
{"level":"error","ts":"2026-11-03T15:27:32.476Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"91f1eec3-a9c6-4908-a09b-a7165cb8b5d4","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-hvml9 is not ready at current generation 1","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:27:32.476Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"91f1eec3-a9c6-4908-a09b-a7165cb8b5d4","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-hvml9 is not ready at current generation 1"}
{"level":"info","ts":"2026-11-03T15:27:32.476Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"91f1eec3-a9c6-4908-a09b-a7165cb8b5d4","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"long-running-hvml9","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:27:32.476Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"91f1eec3-a9c6-4908-a09b-a7165cb8b5d4","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"long-running-hvml9","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:27:32.476Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"91f1eec3-a9c6-4908-a09b-a7165cb8b5d4","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"long-running-hvml9","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:27:32.476Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"91f1eec3-a9c6-4908-a09b-a7165cb8b5d4"}
{"level":"error","ts":"2026-11-03T15:27:32.476Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"91f1eec3-a9c6-4908-a09b-a7165cb8b5d4","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-hvml9 is not ready at current generation 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:27:32.633Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"ed15ba42-10c9-4069-9053-38a56bc11cb5","certificate":"ray-head-cert-long-running-hvml9"}
{"level":"info","ts":"2026-11-03T15:27:32.633Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"ed15ba42-10c9-4069-9053-38a56bc11cb5"}
{"level":"info","ts":"2026-11-03T15:27:32.633Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"3f1a6cfe-ff32-41a8-8ca3-27b0c625191c","cluster":"long-running-hvml9"}
{"level":"info","ts":"2026-11-03T15:27:32.797Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"31258006-a840-4a9a-be89-995c20aa506c"}
{"level":"info","ts":"2026-11-03T15:27:32.797Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"31258006-a840-4a9a-be89-995c20aa506c"}
{"level":"info","ts":"2026-11-03T15:27:32.797Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"31258006-a840-4a9a-be89-995c20aa506c","1 head service found":"long-running-hvml9-head-svc"}
{"level":"info","ts":"2026-11-03T15:27:32.797Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"31258006-a840-4a9a-be89-995c20aa506c"}
{"level":"info","ts":"2026-11-03T15:27:32.848Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"31258006-a840-4a9a-be89-995c20aa506c","error":"certificate ray-head-cert-long-running-hvml9 is not ready at current generation 1"}
{"level":"error","ts":"2026-11-03T15:27:32.848Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"31258006-a840-4a9a-be89-995c20aa506c","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-hvml9 is not ready at current generation 1","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:27:32.848Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"31258006-a840-4a9a-be89-995c20aa506c","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-hvml9 is not ready at current generation 1"}
{"level":"info","ts":"2026-11-03T15:27:32.848Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"31258006-a840-4a9a-be89-995c20aa506c","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"long-running-hvml9","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:27:32.848Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"31258006-a840-4a9a-be89-995c20aa506c","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"long-running-hvml9","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:27:32.848Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"31258006-a840-4a9a-be89-995c20aa506c","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"long-running-hvml9","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:27:32.848Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"31258006-a840-4a9a-be89-995c20aa506c"}
{"level":"error","ts":"2026-11-03T15:27:32.848Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"31258006-a840-4a9a-be89-995c20aa506c","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-hvml9 is not ready at current generation 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:27:33.094Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"3f1a6cfe-ff32-41a8-8ca3-27b0c625191c","certificate":"ray-head-cert-long-running-hvml9"}
{"level":"info","ts":"2026-11-03T15:27:33.094Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"3f1a6cfe-ff32-41a8-8ca3-27b0c625191c"}
{"level":"info","ts":"2026-11-03T15:27:33.489Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"0f4f4c67-cdc3-48de-992e-4aaadc9a577b"}
{"level":"info","ts":"2026-11-03T15:27:33.489Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"0f4f4c67-cdc3-48de-992e-4aaadc9a577b"}
{"level":"info","ts":"2026-11-03T15:27:33.489Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"0f4f4c67-cdc3-48de-992e-4aaadc9a577b","1 head service found":"long-running-hvml9-head-svc"}
{"level":"info","ts":"2026-11-03T15:27:33.489Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"0f4f4c67-cdc3-48de-992e-4aaadc9a577b"}
{"level":"info","ts":"2026-11-03T15:27:33.540Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"0f4f4c67-cdc3-48de-992e-4aaadc9a577b","error":"certificate ray-head-cert-long-running-hvml9 is not ready at current generation 1"}
{"level":"error","ts":"2026-11-03T15:27:33.540Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"0f4f4c67-cdc3-48de-992e-4aaadc9a577b","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-hvml9 is not ready at current generation 1","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:27:33.540Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"0f4f4c67-cdc3-48de-992e-4aaadc9a577b","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-hvml9 is not ready at current generation 1"}
{"level":"info","ts":"2026-11-03T15:27:33.540Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"0f4f4c67-cdc3-48de-992e-4aaadc9a577b","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"long-running-hvml9","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:27:33.540Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"0f4f4c67-cdc3-48de-992e-4aaadc9a577b","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"long-running-hvml9","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:27:33.540Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"0f4f4c67-cdc3-48de-992e-4aaadc9a577b","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"long-running-hvml9","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:27:33.540Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"0f4f4c67-cdc3-48de-992e-4aaadc9a577b"}
{"level":"error","ts":"2026-11-03T15:27:33.540Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"0f4f4c67-cdc3-48de-992e-4aaadc9a577b","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-hvml9 is not ready at current generation 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:27:34.821Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"fc573d0c-1fd8-4cf6-8259-fd2ca36963d6"}
{"level":"info","ts":"2026-11-03T15:27:34.821Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"fc573d0c-1fd8-4cf6-8259-fd2ca36963d6"}
{"level":"info","ts":"2026-11-03T15:27:34.821Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"fc573d0c-1fd8-4cf6-8259-fd2ca36963d6","1 head service found":"long-running-hvml9-head-svc"}
{"level":"info","ts":"2026-11-03T15:27:34.821Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"fc573d0c-1fd8-4cf6-8259-fd2ca36963d6"}
{"level":"info","ts":"2026-11-03T15:27:34.871Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"fc573d0c-1fd8-4cf6-8259-fd2ca36963d6","error":"certificate ray-head-cert-long-running-hvml9 is not ready at current generation 1"}
{"level":"error","ts":"2026-11-03T15:27:34.871Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"fc573d0c-1fd8-4cf6-8259-fd2ca36963d6","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-hvml9 is not ready at current generation 1","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:27:34.871Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"fc573d0c-1fd8-4cf6-8259-fd2ca36963d6","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-hvml9 is not ready at current generation 1"}
{"level":"info","ts":"2026-11-03T15:27:34.871Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"fc573d0c-1fd8-4cf6-8259-fd2ca36963d6","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"long-running-hvml9","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:27:34.872Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"fc573d0c-1fd8-4cf6-8259-fd2ca36963d6","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"long-running-hvml9","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:27:34.872Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"fc573d0c-1fd8-4cf6-8259-fd2ca36963d6","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"long-running-hvml9","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:27:34.872Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"fc573d0c-1fd8-4cf6-8259-fd2ca36963d6"}
{"level":"error","ts":"2026-11-03T15:27:34.872Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"fc573d0c-1fd8-4cf6-8259-fd2ca36963d6","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-hvml9 is not ready at current generation 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:27:35.442Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"9c9153e6-21ff-4b42-89ed-bebb5ba2d891","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:27:35.442Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"9c9153e6-21ff-4b42-89ed-bebb5ba2d891","name":"long-running-hvml9"}
{"level":"info","ts":"2026-11-03T15:27:35.442Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"9c9153e6-21ff-4b42-89ed-bebb5ba2d891","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:27:35.442Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"9c9153e6-21ff-4b42-89ed-bebb5ba2d891"}
{"level":"info","ts":"2026-11-03T15:27:35.442Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"9c9153e6-21ff-4b42-89ed-bebb5ba2d891","RayCluster":"long-running-hvml9","State":""}
{"level":"info","ts":"2026-11-03T15:27:35.442Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"9c9153e6-21ff-4b42-89ed-bebb5ba2d891","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-cdx2h","rayClusterName":"long-running-hvml9","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:31Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-hvml9-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:32Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-cdx2h","rayClusterName":"long-running-hvml9","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:31Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-hvml9-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:32Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:27:37.432Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"e3c3e491-575f-4670-a6c7-10cfc84c5251"}
{"level":"info","ts":"2026-11-03T15:27:37.432Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"e3c3e491-575f-4670-a6c7-10cfc84c5251"}
{"level":"info","ts":"2026-11-03T15:27:37.432Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"e3c3e491-575f-4670-a6c7-10cfc84c5251","1 head service found":"long-running-hvml9-head-svc"}
{"level":"info","ts":"2026-11-03T15:27:37.432Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"e3c3e491-575f-4670-a6c7-10cfc84c5251"}
{"level":"info","ts":"2026-11-03T15:27:37.486Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"e3c3e491-575f-4670-a6c7-10cfc84c5251","error":"certificate ray-head-cert-long-running-hvml9 is not ready at current generation 1"}
{"level":"error","ts":"2026-11-03T15:27:37.486Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"e3c3e491-575f-4670-a6c7-10cfc84c5251","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-hvml9 is not ready at current generation 1","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:27:37.486Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"e3c3e491-575f-4670-a6c7-10cfc84c5251","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-hvml9 is not ready at current generation 1"}
{"level":"info","ts":"2026-11-03T15:27:37.486Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"e3c3e491-575f-4670-a6c7-10cfc84c5251","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"long-running-hvml9","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:27:37.486Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"e3c3e491-575f-4670-a6c7-10cfc84c5251","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"long-running-hvml9","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:27:37.486Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"e3c3e491-575f-4670-a6c7-10cfc84c5251","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"long-running-hvml9","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:27:37.486Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"e3c3e491-575f-4670-a6c7-10cfc84c5251"}
{"level":"error","ts":"2026-11-03T15:27:37.486Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"e3c3e491-575f-4670-a6c7-10cfc84c5251","error":"mTLS secrets not ready: certificate ray-head-cert-long-running-hvml9 is not ready at current generation 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:27:38.444Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"a9cd23a8-be86-4dd0-839f-2a86895cbb41","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:27:38.444Z","logger":"controllers.RayJob","msg":"The RayJob has passed the activeDeadlineSeconds. Transition the status to `Failed`.","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"a9cd23a8-be86-4dd0-839f-2a86895cbb41","StartTime":"2026-11-03 15:27:31 +0000 UTC","ActiveDeadlineSeconds":5}
{"level":"info","ts":"2026-11-03T15:27:38.444Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"a9cd23a8-be86-4dd0-839f-2a86895cbb41","oldRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-cdx2h","rayClusterName":"long-running-hvml9","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:31Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-hvml9-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:32Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"long-running-cdx2h","rayClusterName":"long-running-hvml9","jobDeploymentStatus":"Failed","reason":"DeadlineExceeded","message":"The RayJob has passed the activeDeadlineSeconds. StartTime: 2026-11-03 15:27:31 +0000 UTC. ActiveDeadlineSeconds: 5","startTime":"2026-11-03T15:27:31Z","succeeded":0,"failed":1,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-hvml9-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:32Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:27:38.444Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"a9cd23a8-be86-4dd0-839f-2a86895cbb41","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Failed"}
{"level":"info","ts":"2026-11-03T15:27:38.541Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"75d06f9f-7897-4b1b-844a-f2f36611b757","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:27:38.541Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"75d06f9f-7897-4b1b-844a-f2f36611b757","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Failed","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":600,"Status.endTime":"2026-11-03 15:27:38 +0000 UTC","Now":"2026-11-03T15:27:38.541Z","ShutdownTime":"2026-11-03T15:37:38.000Z"}
{"level":"info","ts":"2026-11-03T15:27:38.541Z","logger":"controllers.RayJob","msg":"TTL has not been met for ShutdownAfterJobFinishes. Requeuing.","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"75d06f9f-7897-4b1b-844a-f2f36611b757","deletionMechanism":"ShutdownAfterJobFinishes","shutdownTime":"2026-11-03T15:37:38.000Z","requeueAfter":601.458384527}
{"level":"info","ts":"2026-11-03T15:27:39.556Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"c47a144a-6c40-4cc5-921a-21c8ce8e2fae","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:27:39.556Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"c47a144a-6c40-4cc5-921a-21c8ce8e2fae","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-11-03T15:27:39.614Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"c47a144a-6c40-4cc5-921a-21c8ce8e2fae"}
{"level":"info","ts":"2026-11-03T15:27:39.614Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"c47a144a-6c40-4cc5-921a-21c8ce8e2fae","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-11-03T15:27:39.614Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"c47a144a-6c40-4cc5-921a-21c8ce8e2fae","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:27:39.614Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"c47a144a-6c40-4cc5-921a-21c8ce8e2fae","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:27:39.672Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"b9dddb6f-5a56-4905-a5cf-34e0f997c9f8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:27:39.672Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"b9dddb6f-5a56-4905-a5cf-34e0f997c9f8","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:27:39.672Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"b9dddb6f-5a56-4905-a5cf-34e0f997c9f8","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:27:39.672Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"b9dddb6f-5a56-4905-a5cf-34e0f997c9f8","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:27:39.728Z","logger":"raycluster-resource","msg":"default","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:27:39.728Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:27:39.786Z","logger":"raycluster-resource","msg":"validate create","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:27:39.840Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"delete-head-after-submit-gg2wh","reconcileID":"d234269b-8102-4bd7-8498-799deefe0d60","namespacedName":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:27:39.840Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"b9dddb6f-5a56-4905-a5cf-34e0f997c9f8","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:27:39.840Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"delete-head-after-submit-gg2wh","reconcileID":"d234269b-8102-4bd7-8498-799deefe0d60","mode":"IntegratedOAuth","cluster":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:27:39.840Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"delete-head-after-submit-gg2wh","reconcileID":"d234269b-8102-4bd7-8498-799deefe0d60","cluster":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:27:39.840Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"delete-head-after-submit-gg2wh","reconcileID":"d234269b-8102-4bd7-8498-799deefe0d60","cluster":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:27:39.840Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"b9dddb6f-5a56-4905-a5cf-34e0f997c9f8"}
{"level":"info","ts":"2026-11-03T15:27:39.840Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"b9dddb6f-5a56-4905-a5cf-34e0f997c9f8","RayCluster":"delete-head-after-submit-gg2wh","State":""}
{"level":"info","ts":"2026-11-03T15:27:39.840Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"b9dddb6f-5a56-4905-a5cf-34e0f997c9f8","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:27:39.840Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"b259511a-b0d9-4fbb-8246-d181c8681225","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:27:39.840Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"b259511a-b0d9-4fbb-8246-d181c8681225","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:27:39.840Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"b259511a-b0d9-4fbb-8246-d181c8681225","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:27:39.840Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"5442375c-3c27-429f-9682-c0a32ebb1057"}
{"level":"info","ts":"2026-11-03T15:27:39.840Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"5442375c-3c27-429f-9682-c0a32ebb1057"}
{"level":"info","ts":"2026-11-03T15:27:39.840Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"b259511a-b0d9-4fbb-8246-d181c8681225"}
{"level":"info","ts":"2026-11-03T15:27:39.840Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"b259511a-b0d9-4fbb-8246-d181c8681225","RayCluster":"delete-head-after-submit-gg2wh","State":""}
{"level":"info","ts":"2026-11-03T15:27:39.840Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"b259511a-b0d9-4fbb-8246-d181c8681225","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:27:39.840Z","logger":"controllers.NetworkPolicy","msg":"Reconciling NetworkPolicies for RayCluster","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"193a7502-d2e6-44c2-bf97-d3e0ee3f46fe","cluster":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:27:39.841Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"3e7f77fa-b938-4912-9a20-d05381378315","cluster":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:27:39.841Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"4060cacc-b682-4dcf-acfc-a480d29ebdfd","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:27:39.841Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"4060cacc-b682-4dcf-acfc-a480d29ebdfd","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:27:39.841Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"4060cacc-b682-4dcf-acfc-a480d29ebdfd","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:27:39.841Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"4060cacc-b682-4dcf-acfc-a480d29ebdfd"}
{"level":"info","ts":"2026-11-03T15:27:39.841Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"4060cacc-b682-4dcf-acfc-a480d29ebdfd","RayCluster":"delete-head-after-submit-gg2wh","State":""}
{"level":"info","ts":"2026-11-03T15:27:39.841Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"4060cacc-b682-4dcf-acfc-a480d29ebdfd","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:27:39.895Z","logger":"controllers.NetworkPolicy","msg":"Successfully created NetworkPolicy","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"193a7502-d2e6-44c2-bf97-d3e0ee3f46fe","name":"delete-head-after-submit-gg2wh-head"}
{"level":"info","ts":"2026-11-03T15:27:39.895Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"5442375c-3c27-429f-9682-c0a32ebb1057","name":"delete-head-after-submit-gg2wh-head-svc"}
{"level":"info","ts":"2026-11-03T15:27:39.895Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"5442375c-3c27-429f-9682-c0a32ebb1057"}
{"level":"info","ts":"2026-11-03T15:27:39.897Z","logger":"raycluster-resource","msg":"default","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:27:39.897Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:27:39.945Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"5442375c-3c27-429f-9682-c0a32ebb1057","error":"certificate ray-head-cert-delete-head-after-submit-gg2wh not found: certificates.cert-manager.io \"ray-head-cert-delete-head-after-submit-gg2wh\" not found"}
{"level":"error","ts":"2026-11-03T15:27:39.945Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"5442375c-3c27-429f-9682-c0a32ebb1057","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-delete-head-after-submit-gg2wh not found: certificates.cert-manager.io \"ray-head-cert-delete-head-after-submit-gg2wh\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:27:39.945Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"5442375c-3c27-429f-9682-c0a32ebb1057","error":"mTLS secrets not ready: certificate ray-head-cert-delete-head-after-submit-gg2wh not found: certificates.cert-manager.io \"ray-head-cert-delete-head-after-submit-gg2wh\" not found"}
{"level":"info","ts":"2026-11-03T15:27:39.945Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"5442375c-3c27-429f-9682-c0a32ebb1057","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"delete-head-after-submit-gg2wh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:27:39.945Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"5442375c-3c27-429f-9682-c0a32ebb1057","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"delete-head-after-submit-gg2wh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:27:39.945Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"5442375c-3c27-429f-9682-c0a32ebb1057","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"delete-head-after-submit-gg2wh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:27:39.945Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"5442375c-3c27-429f-9682-c0a32ebb1057","name":"delete-head-after-submit-gg2wh","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:27:39.950Z","logger":"controllers.NetworkPolicy","msg":"Successfully created NetworkPolicy","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"193a7502-d2e6-44c2-bf97-d3e0ee3f46fe","name":"delete-head-after-submit-gg2wh-workers-small-group"}
{"level":"info","ts":"2026-11-03T15:27:39.950Z","logger":"controllers.NetworkPolicy","msg":"Successfully reconciled NetworkPolicies for RayCluster","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"193a7502-d2e6-44c2-bf97-d3e0ee3f46fe","cluster":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:27:39.950Z","logger":"controllers.NetworkPolicy","msg":"Reconciling NetworkPolicies for RayCluster","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"6b0c0ec5-70d9-48f4-a1bd-96c39616c1f8","cluster":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:27:39.954Z","logger":"raycluster-resource","msg":"validate update","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:27:40.004Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"5442375c-3c27-429f-9682-c0a32ebb1057"}
{"level":"error","ts":"2026-11-03T15:27:40.004Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"5442375c-3c27-429f-9682-c0a32ebb1057","error":"mTLS secrets not ready: certificate ray-head-cert-delete-head-after-submit-gg2wh not found: certificates.cert-manager.io \"ray-head-cert-delete-head-after-submit-gg2wh\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:27:40.004Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"f8b126f3-8466-47db-a741-5e241da2e4e7"}
{"level":"info","ts":"2026-11-03T15:27:40.004Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"f8b126f3-8466-47db-a741-5e241da2e4e7"}
{"level":"info","ts":"2026-11-03T15:27:40.004Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"f8b126f3-8466-47db-a741-5e241da2e4e7","1 head service found":"delete-head-after-submit-gg2wh-head-svc"}
{"level":"info","ts":"2026-11-03T15:27:40.004Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"f8b126f3-8466-47db-a741-5e241da2e4e7"}
{"level":"info","ts":"2026-11-03T15:27:40.005Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"f711e6c1-994c-4fb7-8967-e6a1a7e9da9c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:27:40.005Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"f711e6c1-994c-4fb7-8967-e6a1a7e9da9c","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:27:40.005Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"f711e6c1-994c-4fb7-8967-e6a1a7e9da9c","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:27:40.005Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"f711e6c1-994c-4fb7-8967-e6a1a7e9da9c"}
{"level":"info","ts":"2026-11-03T15:27:40.005Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"f711e6c1-994c-4fb7-8967-e6a1a7e9da9c","RayCluster":"delete-head-after-submit-gg2wh","State":""}
{"level":"info","ts":"2026-11-03T15:27:40.005Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"f711e6c1-994c-4fb7-8967-e6a1a7e9da9c","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:27:40.005Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"f711e6c1-994c-4fb7-8967-e6a1a7e9da9c","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:27:40.015Z","logger":"raycluster-resource","msg":"default","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:27:40.015Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:27:40.054Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"f8b126f3-8466-47db-a741-5e241da2e4e7","error":"certificate ray-head-cert-delete-head-after-submit-gg2wh not found: certificates.cert-manager.io \"ray-head-cert-delete-head-after-submit-gg2wh\" not found"}
{"level":"error","ts":"2026-11-03T15:27:40.054Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"f8b126f3-8466-47db-a741-5e241da2e4e7","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-delete-head-after-submit-gg2wh not found: certificates.cert-manager.io \"ray-head-cert-delete-head-after-submit-gg2wh\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:27:40.054Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"f8b126f3-8466-47db-a741-5e241da2e4e7","error":"mTLS secrets not ready: certificate ray-head-cert-delete-head-after-submit-gg2wh not found: certificates.cert-manager.io \"ray-head-cert-delete-head-after-submit-gg2wh\" not found"}
{"level":"info","ts":"2026-11-03T15:27:40.054Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"f8b126f3-8466-47db-a741-5e241da2e4e7","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"delete-head-after-submit-gg2wh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:27:40.055Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"f8b126f3-8466-47db-a741-5e241da2e4e7","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"delete-head-after-submit-gg2wh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:27:40.055Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"f8b126f3-8466-47db-a741-5e241da2e4e7","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"delete-head-after-submit-gg2wh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:27:40.055Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"f8b126f3-8466-47db-a741-5e241da2e4e7","name":"delete-head-after-submit-gg2wh","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:40Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:40Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:40Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:40Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:40Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:27:40.060Z","logger":"controllers.NetworkPolicy","msg":"Successfully reconciled NetworkPolicies for RayCluster","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"6b0c0ec5-70d9-48f4-a1bd-96c39616c1f8","cluster":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}
{"level":"error","ts":"2026-11-03T15:27:40.064Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"delete-head-after-submit-gg2wh","reconcileID":"d234269b-8102-4bd7-8498-799deefe0d60","error":"failed to add finalizer: Operation cannot be fulfilled on rayclusters.ray.io \"delete-head-after-submit-gg2wh\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:167\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:27:40.064Z","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"delete-head-after-submit-gg2wh","reconcileID":"d234269b-8102-4bd7-8498-799deefe0d60"}
{"level":"error","ts":"2026-11-03T15:27:40.064Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"delete-head-after-submit-gg2wh","reconcileID":"d234269b-8102-4bd7-8498-799deefe0d60","error":"failed to add finalizer: Operation cannot be fulfilled on rayclusters.ray.io \"delete-head-after-submit-gg2wh\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:27:40.064Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"delete-head-after-submit-gg2wh","reconcileID":"442ffbee-83ea-4d9c-8d6a-fdaaf75b613c","namespacedName":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:27:40.064Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"a25fc5e1-c055-4f35-9de6-0fc8f15897a0","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:27:40.064Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"a25fc5e1-c055-4f35-9de6-0fc8f15897a0","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:27:40.064Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"delete-head-after-submit-gg2wh","reconcileID":"442ffbee-83ea-4d9c-8d6a-fdaaf75b613c","mode":"IntegratedOAuth","cluster":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:27:40.064Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"delete-head-after-submit-gg2wh","reconcileID":"442ffbee-83ea-4d9c-8d6a-fdaaf75b613c","cluster":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:27:40.064Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"delete-head-after-submit-gg2wh","reconcileID":"442ffbee-83ea-4d9c-8d6a-fdaaf75b613c","cluster":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:27:40.064Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"a25fc5e1-c055-4f35-9de6-0fc8f15897a0","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:27:40.064Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"a25fc5e1-c055-4f35-9de6-0fc8f15897a0"}
{"level":"info","ts":"2026-11-03T15:27:40.064Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"a25fc5e1-c055-4f35-9de6-0fc8f15897a0","RayCluster":"delete-head-after-submit-gg2wh","State":""}
{"level":"info","ts":"2026-11-03T15:27:40.064Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"a25fc5e1-c055-4f35-9de6-0fc8f15897a0","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:27:40.109Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"f8b126f3-8466-47db-a741-5e241da2e4e7","name":"delete-head-after-submit-gg2wh","error":"Operation cannot be fulfilled on rayclusters.ray.io \"delete-head-after-submit-gg2wh\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-kbx4w","name":"delete-head-after-submit-gg2wh"}}
{"level":"info","ts":"2026-11-03T15:27:40.109Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"f8b126f3-8466-47db-a741-5e241da2e4e7"}
{"level":"error","ts":"2026-11-03T15:27:40.109Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"f8b126f3-8466-47db-a741-5e241da2e4e7","error":"mTLS secrets not ready: certificate ray-head-cert-delete-head-after-submit-gg2wh not found: certificates.cert-manager.io \"ray-head-cert-delete-head-after-submit-gg2wh\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:27:40.119Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"fd27902a-be52-456b-952d-5695af06fea7"}
{"level":"info","ts":"2026-11-03T15:27:40.119Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"fd27902a-be52-456b-952d-5695af06fea7"}
{"level":"info","ts":"2026-11-03T15:27:40.119Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"fd27902a-be52-456b-952d-5695af06fea7","1 head service found":"delete-head-after-submit-gg2wh-head-svc"}
{"level":"info","ts":"2026-11-03T15:27:40.119Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"fd27902a-be52-456b-952d-5695af06fea7"}
{"level":"info","ts":"2026-11-03T15:27:40.122Z","logger":"raycluster-resource","msg":"default","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:27:40.122Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:27:40.169Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"fd27902a-be52-456b-952d-5695af06fea7","error":"certificate ray-head-cert-delete-head-after-submit-gg2wh not found: certificates.cert-manager.io \"ray-head-cert-delete-head-after-submit-gg2wh\" not found"}
{"level":"error","ts":"2026-11-03T15:27:40.169Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"fd27902a-be52-456b-952d-5695af06fea7","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-delete-head-after-submit-gg2wh not found: certificates.cert-manager.io \"ray-head-cert-delete-head-after-submit-gg2wh\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:27:40.169Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"fd27902a-be52-456b-952d-5695af06fea7","error":"mTLS secrets not ready: certificate ray-head-cert-delete-head-after-submit-gg2wh not found: certificates.cert-manager.io \"ray-head-cert-delete-head-after-submit-gg2wh\" not found"}
{"level":"info","ts":"2026-11-03T15:27:40.169Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"fd27902a-be52-456b-952d-5695af06fea7","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"delete-head-after-submit-gg2wh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:27:40.169Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"fd27902a-be52-456b-952d-5695af06fea7","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"delete-head-after-submit-gg2wh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:27:40.169Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"fd27902a-be52-456b-952d-5695af06fea7","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"delete-head-after-submit-gg2wh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:27:40.169Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"fd27902a-be52-456b-952d-5695af06fea7"}
{"level":"error","ts":"2026-11-03T15:27:40.169Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"fd27902a-be52-456b-952d-5695af06fea7","error":"mTLS secrets not ready: certificate ray-head-cert-delete-head-after-submit-gg2wh not found: certificates.cert-manager.io \"ray-head-cert-delete-head-after-submit-gg2wh\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:27:40.180Z","logger":"raycluster-resource","msg":"validate update","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:27:40.189Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"61a2d004-e385-4d35-9fcd-12bd22aab463"}
{"level":"info","ts":"2026-11-03T15:27:40.189Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"61a2d004-e385-4d35-9fcd-12bd22aab463"}
{"level":"info","ts":"2026-11-03T15:27:40.189Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"61a2d004-e385-4d35-9fcd-12bd22aab463","1 head service found":"delete-head-after-submit-gg2wh-head-svc"}
{"level":"info","ts":"2026-11-03T15:27:40.189Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"61a2d004-e385-4d35-9fcd-12bd22aab463"}
{"level":"info","ts":"2026-11-03T15:27:40.235Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"cee04eb8-cdb5-4dc4-8f1c-1b3ef173d842","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:27:40.235Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"cee04eb8-cdb5-4dc4-8f1c-1b3ef173d842","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:27:40.235Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"cee04eb8-cdb5-4dc4-8f1c-1b3ef173d842","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:27:40.235Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"cee04eb8-cdb5-4dc4-8f1c-1b3ef173d842"}
{"level":"info","ts":"2026-11-03T15:27:40.235Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"cee04eb8-cdb5-4dc4-8f1c-1b3ef173d842","RayCluster":"delete-head-after-submit-gg2wh","State":""}
{"level":"info","ts":"2026-11-03T15:27:40.235Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"cee04eb8-cdb5-4dc4-8f1c-1b3ef173d842","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:27:40.239Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"61a2d004-e385-4d35-9fcd-12bd22aab463","error":"certificate ray-head-cert-delete-head-after-submit-gg2wh not found: certificates.cert-manager.io \"ray-head-cert-delete-head-after-submit-gg2wh\" not found"}
{"level":"error","ts":"2026-11-03T15:27:40.239Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"61a2d004-e385-4d35-9fcd-12bd22aab463","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-delete-head-after-submit-gg2wh not found: certificates.cert-manager.io \"ray-head-cert-delete-head-after-submit-gg2wh\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:27:40.239Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"61a2d004-e385-4d35-9fcd-12bd22aab463","error":"mTLS secrets not ready: certificate ray-head-cert-delete-head-after-submit-gg2wh not found: certificates.cert-manager.io \"ray-head-cert-delete-head-after-submit-gg2wh\" not found"}
{"level":"info","ts":"2026-11-03T15:27:40.239Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"61a2d004-e385-4d35-9fcd-12bd22aab463","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"delete-head-after-submit-gg2wh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:27:40.239Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"61a2d004-e385-4d35-9fcd-12bd22aab463","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"delete-head-after-submit-gg2wh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:27:40.240Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"61a2d004-e385-4d35-9fcd-12bd22aab463","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"delete-head-after-submit-gg2wh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:27:40.240Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"61a2d004-e385-4d35-9fcd-12bd22aab463"}
{"level":"error","ts":"2026-11-03T15:27:40.240Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"61a2d004-e385-4d35-9fcd-12bd22aab463","error":"mTLS secrets not ready: certificate ray-head-cert-delete-head-after-submit-gg2wh not found: certificates.cert-manager.io \"ray-head-cert-delete-head-after-submit-gg2wh\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:27:40.280Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"49e2b00a-d87a-4744-984b-e959662b6a55"}
{"level":"info","ts":"2026-11-03T15:27:40.280Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"49e2b00a-d87a-4744-984b-e959662b6a55"}
{"level":"info","ts":"2026-11-03T15:27:40.280Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"49e2b00a-d87a-4744-984b-e959662b6a55","1 head service found":"delete-head-after-submit-gg2wh-head-svc"}
{"level":"info","ts":"2026-11-03T15:27:40.280Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"49e2b00a-d87a-4744-984b-e959662b6a55"}
{"level":"info","ts":"2026-11-03T15:27:40.287Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"delete-head-after-submit-gg2wh","reconcileID":"442ffbee-83ea-4d9c-8d6a-fdaaf75b613c","name":"delete-head-after-submit-gg2wh-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-11-03T15:27:40.330Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"49e2b00a-d87a-4744-984b-e959662b6a55","error":"certificate ray-head-cert-delete-head-after-submit-gg2wh not found: certificates.cert-manager.io \"ray-head-cert-delete-head-after-submit-gg2wh\" not found"}
{"level":"error","ts":"2026-11-03T15:27:40.330Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"49e2b00a-d87a-4744-984b-e959662b6a55","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-delete-head-after-submit-gg2wh not found: certificates.cert-manager.io \"ray-head-cert-delete-head-after-submit-gg2wh\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:27:40.330Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"49e2b00a-d87a-4744-984b-e959662b6a55","error":"mTLS secrets not ready: certificate ray-head-cert-delete-head-after-submit-gg2wh not found: certificates.cert-manager.io \"ray-head-cert-delete-head-after-submit-gg2wh\" not found"}
{"level":"info","ts":"2026-11-03T15:27:40.330Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"49e2b00a-d87a-4744-984b-e959662b6a55","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"delete-head-after-submit-gg2wh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:27:40.330Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"49e2b00a-d87a-4744-984b-e959662b6a55","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"delete-head-after-submit-gg2wh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:27:40.330Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"49e2b00a-d87a-4744-984b-e959662b6a55","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"delete-head-after-submit-gg2wh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:27:40.330Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"49e2b00a-d87a-4744-984b-e959662b6a55"}
{"level":"error","ts":"2026-11-03T15:27:40.330Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"49e2b00a-d87a-4744-984b-e959662b6a55","error":"mTLS secrets not ready: certificate ray-head-cert-delete-head-after-submit-gg2wh not found: certificates.cert-manager.io \"ray-head-cert-delete-head-after-submit-gg2wh\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:27:40.342Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"delete-head-after-submit-gg2wh","reconcileID":"442ffbee-83ea-4d9c-8d6a-fdaaf75b613c","name":"test-ns-kbx4w-delete-head-after-submit-gg2wh","operation":"created"}
{"level":"info","ts":"2026-11-03T15:27:40.395Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"delete-head-after-submit-gg2wh","reconcileID":"442ffbee-83ea-4d9c-8d6a-fdaaf75b613c","name":"kube-rbac-proxy-config-delete-head-after-submit-gg2wh","operation":"created"}
{"level":"info","ts":"2026-11-03T15:27:40.411Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"32738bfc-5e48-4286-894e-9f5e261ca453"}
{"level":"info","ts":"2026-11-03T15:27:40.411Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"32738bfc-5e48-4286-894e-9f5e261ca453"}
{"level":"info","ts":"2026-11-03T15:27:40.411Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"32738bfc-5e48-4286-894e-9f5e261ca453","1 head service found":"delete-head-after-submit-gg2wh-head-svc"}
{"level":"info","ts":"2026-11-03T15:27:40.411Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"32738bfc-5e48-4286-894e-9f5e261ca453"}
{"level":"error","ts":"2026-11-03T15:27:40.450Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"delete-head-after-submit-gg2wh","reconcileID":"442ffbee-83ea-4d9c-8d6a-fdaaf75b613c","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"delete-head-after-submit-gg2wh\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:167\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:27:40.450Z","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"delete-head-after-submit-gg2wh","reconcileID":"442ffbee-83ea-4d9c-8d6a-fdaaf75b613c"}
{"level":"error","ts":"2026-11-03T15:27:40.450Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"delete-head-after-submit-gg2wh","reconcileID":"442ffbee-83ea-4d9c-8d6a-fdaaf75b613c","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"delete-head-after-submit-gg2wh\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:27:40.450Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"delete-head-after-submit-gg2wh","reconcileID":"d71d13f2-44e6-4984-b627-0202f85c47ff","namespacedName":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:27:40.450Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"delete-head-after-submit-gg2wh","reconcileID":"d71d13f2-44e6-4984-b627-0202f85c47ff","mode":"IntegratedOAuth","cluster":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:27:40.450Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"delete-head-after-submit-gg2wh","reconcileID":"d71d13f2-44e6-4984-b627-0202f85c47ff","cluster":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:27:40.461Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"32738bfc-5e48-4286-894e-9f5e261ca453","error":"certificate ray-head-cert-delete-head-after-submit-gg2wh not found: certificates.cert-manager.io \"ray-head-cert-delete-head-after-submit-gg2wh\" not found"}
{"level":"error","ts":"2026-11-03T15:27:40.462Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"32738bfc-5e48-4286-894e-9f5e261ca453","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-delete-head-after-submit-gg2wh not found: certificates.cert-manager.io \"ray-head-cert-delete-head-after-submit-gg2wh\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:27:40.462Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"32738bfc-5e48-4286-894e-9f5e261ca453","error":"mTLS secrets not ready: certificate ray-head-cert-delete-head-after-submit-gg2wh not found: certificates.cert-manager.io \"ray-head-cert-delete-head-after-submit-gg2wh\" not found"}
{"level":"info","ts":"2026-11-03T15:27:40.462Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"32738bfc-5e48-4286-894e-9f5e261ca453","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"delete-head-after-submit-gg2wh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:27:40.462Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"32738bfc-5e48-4286-894e-9f5e261ca453","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"delete-head-after-submit-gg2wh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:27:40.462Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"32738bfc-5e48-4286-894e-9f5e261ca453","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"delete-head-after-submit-gg2wh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:27:40.462Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"32738bfc-5e48-4286-894e-9f5e261ca453"}
{"level":"error","ts":"2026-11-03T15:27:40.462Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"32738bfc-5e48-4286-894e-9f5e261ca453","error":"mTLS secrets not ready: certificate ray-head-cert-delete-head-after-submit-gg2wh not found: certificates.cert-manager.io \"ray-head-cert-delete-head-after-submit-gg2wh\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:27:40.511Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"delete-head-after-submit-gg2wh","reconcileID":"d71d13f2-44e6-4984-b627-0202f85c47ff","cluster":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:27:40.511Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"73980b6d-7dcb-4980-9dff-15dd7973dbf0","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:27:40.511Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"73980b6d-7dcb-4980-9dff-15dd7973dbf0","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:27:40.511Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"73980b6d-7dcb-4980-9dff-15dd7973dbf0","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:27:40.512Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"73980b6d-7dcb-4980-9dff-15dd7973dbf0"}
{"level":"info","ts":"2026-11-03T15:27:40.512Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"73980b6d-7dcb-4980-9dff-15dd7973dbf0","RayCluster":"delete-head-after-submit-gg2wh","State":""}
{"level":"info","ts":"2026-11-03T15:27:40.512Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"73980b6d-7dcb-4980-9dff-15dd7973dbf0","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:27:40.512Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"73980b6d-7dcb-4980-9dff-15dd7973dbf0","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:27:40.572Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"04305671-ac16-423d-8d9b-49848b78dbb9","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:27:40.572Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"04305671-ac16-423d-8d9b-49848b78dbb9","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:27:40.572Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"04305671-ac16-423d-8d9b-49848b78dbb9","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:27:40.572Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"04305671-ac16-423d-8d9b-49848b78dbb9"}
{"level":"info","ts":"2026-11-03T15:27:40.572Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"04305671-ac16-423d-8d9b-49848b78dbb9","RayCluster":"delete-head-after-submit-gg2wh","State":""}
{"level":"info","ts":"2026-11-03T15:27:40.572Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"04305671-ac16-423d-8d9b-49848b78dbb9","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:27:40.623Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"695a9222-99c4-4eef-a053-dfdf11106e96"}
{"level":"info","ts":"2026-11-03T15:27:40.623Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"695a9222-99c4-4eef-a053-dfdf11106e96"}
{"level":"info","ts":"2026-11-03T15:27:40.623Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"695a9222-99c4-4eef-a053-dfdf11106e96","1 head service found":"delete-head-after-submit-gg2wh-head-svc"}
{"level":"info","ts":"2026-11-03T15:27:40.623Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"695a9222-99c4-4eef-a053-dfdf11106e96"}
{"level":"info","ts":"2026-11-03T15:27:40.674Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"695a9222-99c4-4eef-a053-dfdf11106e96","error":"certificate ray-head-cert-delete-head-after-submit-gg2wh is not ready at current generation 1"}
{"level":"error","ts":"2026-11-03T15:27:40.674Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"695a9222-99c4-4eef-a053-dfdf11106e96","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-delete-head-after-submit-gg2wh is not ready at current generation 1","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:27:40.674Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"695a9222-99c4-4eef-a053-dfdf11106e96","error":"mTLS secrets not ready: certificate ray-head-cert-delete-head-after-submit-gg2wh is not ready at current generation 1"}
{"level":"info","ts":"2026-11-03T15:27:40.674Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"695a9222-99c4-4eef-a053-dfdf11106e96","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"delete-head-after-submit-gg2wh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:27:40.674Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"695a9222-99c4-4eef-a053-dfdf11106e96","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"delete-head-after-submit-gg2wh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:27:40.674Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"695a9222-99c4-4eef-a053-dfdf11106e96","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"delete-head-after-submit-gg2wh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:27:40.674Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"695a9222-99c4-4eef-a053-dfdf11106e96"}
{"level":"error","ts":"2026-11-03T15:27:40.674Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"695a9222-99c4-4eef-a053-dfdf11106e96","error":"mTLS secrets not ready: certificate ray-head-cert-delete-head-after-submit-gg2wh is not ready at current generation 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:27:40.731Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"3e7f77fa-b938-4912-9a20-d05381378315","certificate":"ray-head-cert-delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:27:40.731Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"3e7f77fa-b938-4912-9a20-d05381378315"}
{"level":"info","ts":"2026-11-03T15:27:40.731Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"1d1ec8b9-ae03-44ab-8355-e0acaafe2066","cluster":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:27:40.995Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"46fd42a3-ca09-4100-b0ac-3173a94585f8"}
{"level":"info","ts":"2026-11-03T15:27:40.995Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"46fd42a3-ca09-4100-b0ac-3173a94585f8"}
{"level":"info","ts":"2026-11-03T15:27:40.995Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"46fd42a3-ca09-4100-b0ac-3173a94585f8","1 head service found":"delete-head-after-submit-gg2wh-head-svc"}
{"level":"info","ts":"2026-11-03T15:27:40.995Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"46fd42a3-ca09-4100-b0ac-3173a94585f8"}
{"level":"info","ts":"2026-11-03T15:27:41.046Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"46fd42a3-ca09-4100-b0ac-3173a94585f8","error":"certificate ray-head-cert-delete-head-after-submit-gg2wh is not ready at current generation 1"}
{"level":"error","ts":"2026-11-03T15:27:41.046Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"46fd42a3-ca09-4100-b0ac-3173a94585f8","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-delete-head-after-submit-gg2wh is not ready at current generation 1","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:27:41.046Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"46fd42a3-ca09-4100-b0ac-3173a94585f8","error":"mTLS secrets not ready: certificate ray-head-cert-delete-head-after-submit-gg2wh is not ready at current generation 1"}
{"level":"info","ts":"2026-11-03T15:27:41.046Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"46fd42a3-ca09-4100-b0ac-3173a94585f8","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"delete-head-after-submit-gg2wh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:27:41.046Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"46fd42a3-ca09-4100-b0ac-3173a94585f8","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"delete-head-after-submit-gg2wh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:27:41.046Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"46fd42a3-ca09-4100-b0ac-3173a94585f8","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"delete-head-after-submit-gg2wh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:27:41.046Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"46fd42a3-ca09-4100-b0ac-3173a94585f8"}
{"level":"error","ts":"2026-11-03T15:27:41.046Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"46fd42a3-ca09-4100-b0ac-3173a94585f8","error":"mTLS secrets not ready: certificate ray-head-cert-delete-head-after-submit-gg2wh is not ready at current generation 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:27:41.186Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"1d1ec8b9-ae03-44ab-8355-e0acaafe2066","certificate":"ray-head-cert-delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:27:41.186Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"1d1ec8b9-ae03-44ab-8355-e0acaafe2066"}
{"level":"info","ts":"2026-11-03T15:27:41.687Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"a70c29f7-0730-4d04-b0bf-41cd53b085fd"}
{"level":"info","ts":"2026-11-03T15:27:41.687Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"a70c29f7-0730-4d04-b0bf-41cd53b085fd"}
{"level":"info","ts":"2026-11-03T15:27:41.687Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"a70c29f7-0730-4d04-b0bf-41cd53b085fd","1 head service found":"delete-head-after-submit-gg2wh-head-svc"}
{"level":"info","ts":"2026-11-03T15:27:41.687Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"a70c29f7-0730-4d04-b0bf-41cd53b085fd"}
{"level":"info","ts":"2026-11-03T15:27:41.738Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"a70c29f7-0730-4d04-b0bf-41cd53b085fd","error":"certificate ray-head-cert-delete-head-after-submit-gg2wh is not ready at current generation 1"}
{"level":"error","ts":"2026-11-03T15:27:41.738Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"a70c29f7-0730-4d04-b0bf-41cd53b085fd","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-delete-head-after-submit-gg2wh is not ready at current generation 1","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:27:41.738Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"a70c29f7-0730-4d04-b0bf-41cd53b085fd","error":"mTLS secrets not ready: certificate ray-head-cert-delete-head-after-submit-gg2wh is not ready at current generation 1"}
{"level":"info","ts":"2026-11-03T15:27:41.738Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"a70c29f7-0730-4d04-b0bf-41cd53b085fd","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"delete-head-after-submit-gg2wh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:27:41.738Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"a70c29f7-0730-4d04-b0bf-41cd53b085fd","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"delete-head-after-submit-gg2wh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:27:41.738Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"a70c29f7-0730-4d04-b0bf-41cd53b085fd","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"delete-head-after-submit-gg2wh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:27:41.738Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"a70c29f7-0730-4d04-b0bf-41cd53b085fd"}
{"level":"error","ts":"2026-11-03T15:27:41.738Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"a70c29f7-0730-4d04-b0bf-41cd53b085fd","error":"mTLS secrets not ready: certificate ray-head-cert-delete-head-after-submit-gg2wh is not ready at current generation 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:27:42.607Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"b6b4ae02-2ae2-4f0e-b79d-2bce0bfc8e1e"}
{"level":"info","ts":"2026-11-03T15:27:42.607Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"b6b4ae02-2ae2-4f0e-b79d-2bce0bfc8e1e"}
{"level":"info","ts":"2026-11-03T15:27:42.607Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"b6b4ae02-2ae2-4f0e-b79d-2bce0bfc8e1e","1 head service found":"long-running-hvml9-head-svc"}
{"level":"info","ts":"2026-11-03T15:27:42.607Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"b6b4ae02-2ae2-4f0e-b79d-2bce0bfc8e1e"}
{"level":"info","ts":"2026-11-03T15:27:42.709Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"b6b4ae02-2ae2-4f0e-b79d-2bce0bfc8e1e","mode":"IntegratedOAuth","cluster":"long-running-hvml9"}
{"level":"info","ts":"2026-11-03T15:27:42.709Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"b6b4ae02-2ae2-4f0e-b79d-2bce0bfc8e1e","cluster":"long-running-hvml9","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-11-03T15:27:42.709Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"b6b4ae02-2ae2-4f0e-b79d-2bce0bfc8e1e","cluster":"long-running-hvml9","authType":"IntegratedOAuth","serviceAccount":"long-running-hvml9-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-11-03T15:27:42.709Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"b6b4ae02-2ae2-4f0e-b79d-2bce0bfc8e1e","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"long-running-hvml9","ray.io/group":"headgroup","ray.io/identifier":"long-running-hvml9-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-11-03T15:27:42.709Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"b6b4ae02-2ae2-4f0e-b79d-2bce0bfc8e1e","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-11-03T15:27:42.709Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"b6b4ae02-2ae2-4f0e-b79d-2bce0bfc8e1e","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-11-03T15:27:42.709Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"b6b4ae02-2ae2-4f0e-b79d-2bce0bfc8e1e","rayNodeType":"head","generatedCmd":"ulimit -n ${RAY_START_ULIMIT_OPEN_FILES:-65536}; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-11-03T15:27:42.709Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"b6b4ae02-2ae2-4f0e-b79d-2bce0bfc8e1e","enabled":true}
{"level":"info","ts":"2026-11-03T15:27:42.822Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"b6b4ae02-2ae2-4f0e-b79d-2bce0bfc8e1e","name":"long-running-hvml9-head-s8j5v"}
{"level":"info","ts":"2026-11-03T15:27:42.822Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"b6b4ae02-2ae2-4f0e-b79d-2bce0bfc8e1e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:27:42.822Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"b6b4ae02-2ae2-4f0e-b79d-2bce0bfc8e1e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:27:42.822Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"b6b4ae02-2ae2-4f0e-b79d-2bce0bfc8e1e","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-11-03T15:27:42.822Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"b6b4ae02-2ae2-4f0e-b79d-2bce0bfc8e1e","group":"small-group","indices":{}}
{"level":"info","ts":"2026-11-03T15:27:42.822Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"b6b4ae02-2ae2-4f0e-b79d-2bce0bfc8e1e","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-11-03T15:27:42.923Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"b6b4ae02-2ae2-4f0e-b79d-2bce0bfc8e1e","creating worker for group":"small-group","index":0,"total":1,"replicaIndex":0}
{"level":"info","ts":"2026-11-03T15:27:42.923Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"b6b4ae02-2ae2-4f0e-b79d-2bce0bfc8e1e","nodeType":"worker","rayStartParams":{"address":"long-running-hvml9-head-svc.test-ns-kbx4w.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-11-03T15:27:42.923Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"b6b4ae02-2ae2-4f0e-b79d-2bce0bfc8e1e","rayStartCmd":"ray start  --address=long-running-hvml9-head-svc.test-ns-kbx4w.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-11-03T15:27:42.923Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"b6b4ae02-2ae2-4f0e-b79d-2bce0bfc8e1e","rayNodeType":"worker","generatedCmd":"ulimit -n ${RAY_START_ULIMIT_OPEN_FILES:-65536}; ray start  --address=long-running-hvml9-head-svc.test-ns-kbx4w.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-11-03T15:27:42.923Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"b6b4ae02-2ae2-4f0e-b79d-2bce0bfc8e1e","enabled":true}
{"level":"info","ts":"2026-11-03T15:27:42.985Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"b6b4ae02-2ae2-4f0e-b79d-2bce0bfc8e1e","name":"long-running-hvml9-small-group-worker-mjl4n"}
{"level":"info","ts":"2026-11-03T15:27:42.985Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"b6b4ae02-2ae2-4f0e-b79d-2bce0bfc8e1e","podName":"long-running-hvml9-head-s8j5v","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:27:42.985Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"b6b4ae02-2ae2-4f0e-b79d-2bce0bfc8e1e","name":"long-running-hvml9","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-hvml9-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:32Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:42Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-hvml9-head-s8j5v","serviceName":"long-running-hvml9-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:32Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:27:43.046Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"c17dc37a-b55a-408e-9120-c3e7bd1c22bf"}
{"level":"info","ts":"2026-11-03T15:27:43.046Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"c17dc37a-b55a-408e-9120-c3e7bd1c22bf"}
{"level":"info","ts":"2026-11-03T15:27:43.046Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"c17dc37a-b55a-408e-9120-c3e7bd1c22bf","1 head service found":"long-running-hvml9-head-svc"}
{"level":"info","ts":"2026-11-03T15:27:43.046Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"c17dc37a-b55a-408e-9120-c3e7bd1c22bf","Found 1 head Pod":"long-running-hvml9-head-s8j5v","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:27:43.046Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"c17dc37a-b55a-408e-9120-c3e7bd1c22bf","head Pod":"long-running-hvml9-head-s8j5v","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-hvml9-head-s8j5v. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:27:43.046Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"c17dc37a-b55a-408e-9120-c3e7bd1c22bf","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:27:43.047Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"c17dc37a-b55a-408e-9120-c3e7bd1c22bf","worker Pod":"long-running-hvml9-small-group-worker-mjl4n","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-hvml9-small-group-worker-mjl4n. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:27:43.047Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"c17dc37a-b55a-408e-9120-c3e7bd1c22bf","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:27:43.047Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"c17dc37a-b55a-408e-9120-c3e7bd1c22bf","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:27:43.047Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"c17dc37a-b55a-408e-9120-c3e7bd1c22bf","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:27:43.047Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"c17dc37a-b55a-408e-9120-c3e7bd1c22bf","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:27:43.047Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"c17dc37a-b55a-408e-9120-c3e7bd1c22bf","podName":"long-running-hvml9-head-s8j5v","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:27:43.047Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"c17dc37a-b55a-408e-9120-c3e7bd1c22bf","podName":"long-running-hvml9-head-s8j5v","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:27:43.047Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"c2bf4d34-09a6-4348-8c9d-43e18c2a775b","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:27:43.047Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"c2bf4d34-09a6-4348-8c9d-43e18c2a775b","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Failed","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":600,"Status.endTime":"2026-11-03 15:27:38 +0000 UTC","Now":"2026-11-03T15:27:43.047Z","ShutdownTime":"2026-11-03T15:37:38.000Z"}
{"level":"info","ts":"2026-11-03T15:27:43.047Z","logger":"controllers.RayJob","msg":"TTL has not been met for ShutdownAfterJobFinishes. Requeuing.","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"c2bf4d34-09a6-4348-8c9d-43e18c2a775b","deletionMechanism":"ShutdownAfterJobFinishes","shutdownTime":"2026-11-03T15:37:38.000Z","requeueAfter":596.952797369}
{"level":"info","ts":"2026-11-03T15:27:43.047Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"c17dc37a-b55a-408e-9120-c3e7bd1c22bf","name":"long-running-hvml9","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:31Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"long-running-hvml9-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:32Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:43Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-hvml9-head-s8j5v","serviceName":"long-running-hvml9-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:32Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:27:43.047Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"ff3aa8ba-ff14-4094-9eee-1337096c9f16","cluster":"long-running-hvml9"}
{"level":"info","ts":"2026-11-03T15:27:43.102Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"c17dc37a-b55a-408e-9120-c3e7bd1c22bf","name":"long-running-hvml9","error":"Operation cannot be fulfilled on rayclusters.ray.io \"long-running-hvml9\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-kbx4w","name":"long-running-hvml9"}}
{"level":"info","ts":"2026-11-03T15:27:43.102Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"c17dc37a-b55a-408e-9120-c3e7bd1c22bf"}
{"level":"error","ts":"2026-11-03T15:27:43.102Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"c17dc37a-b55a-408e-9120-c3e7bd1c22bf","error":"Operation cannot be fulfilled on rayclusters.ray.io \"long-running-hvml9\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:27:43.102Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"f9909f6b-aff3-4180-bcb4-105342fb45a8"}
{"level":"info","ts":"2026-11-03T15:27:43.102Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"f9909f6b-aff3-4180-bcb4-105342fb45a8"}
{"level":"info","ts":"2026-11-03T15:27:43.102Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"f9909f6b-aff3-4180-bcb4-105342fb45a8","1 head service found":"delete-head-after-submit-gg2wh-head-svc"}
{"level":"info","ts":"2026-11-03T15:27:43.102Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"f9909f6b-aff3-4180-bcb4-105342fb45a8"}
{"level":"info","ts":"2026-11-03T15:27:43.203Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"f9909f6b-aff3-4180-bcb4-105342fb45a8","mode":"IntegratedOAuth","cluster":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:27:43.203Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"f9909f6b-aff3-4180-bcb4-105342fb45a8","cluster":"delete-head-after-submit-gg2wh","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-11-03T15:27:43.203Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"f9909f6b-aff3-4180-bcb4-105342fb45a8","cluster":"delete-head-after-submit-gg2wh","authType":"IntegratedOAuth","serviceAccount":"delete-head-after-submit-gg2wh-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-11-03T15:27:43.203Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"f9909f6b-aff3-4180-bcb4-105342fb45a8","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"delete-head-after-submit-gg2wh","ray.io/group":"headgroup","ray.io/identifier":"delete-head-after-submit-gg2wh-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-11-03T15:27:43.203Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"f9909f6b-aff3-4180-bcb4-105342fb45a8","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-11-03T15:27:43.203Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"f9909f6b-aff3-4180-bcb4-105342fb45a8","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-11-03T15:27:43.203Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"f9909f6b-aff3-4180-bcb4-105342fb45a8","rayNodeType":"head","generatedCmd":"ulimit -n ${RAY_START_ULIMIT_OPEN_FILES:-65536}; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-11-03T15:27:43.203Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"f9909f6b-aff3-4180-bcb4-105342fb45a8","enabled":true}
{"level":"info","ts":"2026-11-03T15:27:43.265Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"f9909f6b-aff3-4180-bcb4-105342fb45a8","name":"delete-head-after-submit-gg2wh-head-b99bx"}
{"level":"info","ts":"2026-11-03T15:27:43.265Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"f9909f6b-aff3-4180-bcb4-105342fb45a8","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:27:43.265Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"f9909f6b-aff3-4180-bcb4-105342fb45a8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:27:43.265Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"f9909f6b-aff3-4180-bcb4-105342fb45a8","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-11-03T15:27:43.265Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"f9909f6b-aff3-4180-bcb4-105342fb45a8","group":"small-group","indices":{}}
{"level":"info","ts":"2026-11-03T15:27:43.265Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"f9909f6b-aff3-4180-bcb4-105342fb45a8","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-11-03T15:27:43.366Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"f9909f6b-aff3-4180-bcb4-105342fb45a8","creating worker for group":"small-group","index":0,"total":1,"replicaIndex":0}
{"level":"info","ts":"2026-11-03T15:27:43.366Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"f9909f6b-aff3-4180-bcb4-105342fb45a8","nodeType":"worker","rayStartParams":{"address":"delete-head-after-submit-gg2wh-head-svc.test-ns-kbx4w.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-11-03T15:27:43.366Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"f9909f6b-aff3-4180-bcb4-105342fb45a8","rayStartCmd":"ray start  --address=delete-head-after-submit-gg2wh-head-svc.test-ns-kbx4w.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-11-03T15:27:43.366Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"f9909f6b-aff3-4180-bcb4-105342fb45a8","rayNodeType":"worker","generatedCmd":"ulimit -n ${RAY_START_ULIMIT_OPEN_FILES:-65536}; ray start  --address=delete-head-after-submit-gg2wh-head-svc.test-ns-kbx4w.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-11-03T15:27:43.366Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"f9909f6b-aff3-4180-bcb4-105342fb45a8","enabled":true}
{"level":"info","ts":"2026-11-03T15:27:43.428Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"f9909f6b-aff3-4180-bcb4-105342fb45a8","name":"delete-head-after-submit-gg2wh-small-group-worker-dxpfb"}
{"level":"info","ts":"2026-11-03T15:27:43.428Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"f9909f6b-aff3-4180-bcb4-105342fb45a8","podName":"delete-head-after-submit-gg2wh-head-b99bx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:27:43.428Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"f9909f6b-aff3-4180-bcb4-105342fb45a8","podName":"delete-head-after-submit-gg2wh-head-b99bx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:27:43.428Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"f9909f6b-aff3-4180-bcb4-105342fb45a8","name":"delete-head-after-submit-gg2wh","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:43Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:27:43.490Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"a48dc4d4-263c-4a06-aed1-c065076624a6"}
{"level":"info","ts":"2026-11-03T15:27:43.490Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"a48dc4d4-263c-4a06-aed1-c065076624a6"}
{"level":"info","ts":"2026-11-03T15:27:43.490Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"a48dc4d4-263c-4a06-aed1-c065076624a6","1 head service found":"long-running-hvml9-head-svc"}
{"level":"info","ts":"2026-11-03T15:27:43.490Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"a48dc4d4-263c-4a06-aed1-c065076624a6","Found 1 head Pod":"long-running-hvml9-head-s8j5v","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:27:43.490Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"75b06190-a465-4636-a09e-03ec7adca9c4","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:27:43.490Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"a48dc4d4-263c-4a06-aed1-c065076624a6","head Pod":"long-running-hvml9-head-s8j5v","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-hvml9-head-s8j5v. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:27:43.490Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"75b06190-a465-4636-a09e-03ec7adca9c4","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:27:43.490Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"a48dc4d4-263c-4a06-aed1-c065076624a6","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:27:43.490Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"75b06190-a465-4636-a09e-03ec7adca9c4","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:27:43.490Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"75b06190-a465-4636-a09e-03ec7adca9c4"}
{"level":"info","ts":"2026-11-03T15:27:43.490Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"75b06190-a465-4636-a09e-03ec7adca9c4","RayCluster":"delete-head-after-submit-gg2wh","State":""}
{"level":"info","ts":"2026-11-03T15:27:43.490Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"a48dc4d4-263c-4a06-aed1-c065076624a6","worker Pod":"long-running-hvml9-small-group-worker-mjl4n","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-hvml9-small-group-worker-mjl4n. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:27:43.490Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"a48dc4d4-263c-4a06-aed1-c065076624a6","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:27:43.490Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"a48dc4d4-263c-4a06-aed1-c065076624a6","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:27:43.490Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"a48dc4d4-263c-4a06-aed1-c065076624a6","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:27:43.491Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"a48dc4d4-263c-4a06-aed1-c065076624a6","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:27:43.490Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"75b06190-a465-4636-a09e-03ec7adca9c4","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:39Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:43Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:27:43.491Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"75b06190-a465-4636-a09e-03ec7adca9c4","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:27:43.491Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"a48dc4d4-263c-4a06-aed1-c065076624a6","podName":"long-running-hvml9-small-group-worker-mjl4n","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:27:43.491Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"a48dc4d4-263c-4a06-aed1-c065076624a6","podName":"long-running-hvml9-small-group-worker-mjl4n","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:27:43.491Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"a48dc4d4-263c-4a06-aed1-c065076624a6","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:27:43.491Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"a48dc4d4-263c-4a06-aed1-c065076624a6","seconds":300}
{"level":"info","ts":"2026-11-03T15:27:43.491Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"ac8b9a22-ceb0-405e-8082-dd31a783969a"}
{"level":"info","ts":"2026-11-03T15:27:43.491Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"ac8b9a22-ceb0-405e-8082-dd31a783969a"}
{"level":"info","ts":"2026-11-03T15:27:43.491Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"ac8b9a22-ceb0-405e-8082-dd31a783969a","1 head service found":"delete-head-after-submit-gg2wh-head-svc"}
{"level":"info","ts":"2026-11-03T15:27:43.491Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"ac8b9a22-ceb0-405e-8082-dd31a783969a","Found 1 head Pod":"delete-head-after-submit-gg2wh-head-b99bx","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:27:43.491Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"ac8b9a22-ceb0-405e-8082-dd31a783969a","head Pod":"delete-head-after-submit-gg2wh-head-b99bx","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod delete-head-after-submit-gg2wh-head-b99bx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:27:43.491Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"ac8b9a22-ceb0-405e-8082-dd31a783969a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:27:43.491Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"ac8b9a22-ceb0-405e-8082-dd31a783969a","worker Pod":"delete-head-after-submit-gg2wh-small-group-worker-dxpfb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod delete-head-after-submit-gg2wh-small-group-worker-dxpfb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:27:43.491Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"ac8b9a22-ceb0-405e-8082-dd31a783969a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:27:43.491Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"ac8b9a22-ceb0-405e-8082-dd31a783969a","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:27:43.491Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"ac8b9a22-ceb0-405e-8082-dd31a783969a","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:27:43.491Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"ac8b9a22-ceb0-405e-8082-dd31a783969a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:27:43.491Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"ac8b9a22-ceb0-405e-8082-dd31a783969a","podName":"delete-head-after-submit-gg2wh-head-b99bx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:27:43.491Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"ac8b9a22-ceb0-405e-8082-dd31a783969a","podName":"delete-head-after-submit-gg2wh-head-b99bx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:27:43.491Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"ac8b9a22-ceb0-405e-8082-dd31a783969a","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:27:43.491Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"ac8b9a22-ceb0-405e-8082-dd31a783969a","seconds":300}
{"level":"info","ts":"2026-11-03T15:27:43.505Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"987289b5-5f7b-4521-82c0-e2c23fb17fde"}
{"level":"info","ts":"2026-11-03T15:27:43.505Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"987289b5-5f7b-4521-82c0-e2c23fb17fde"}
{"level":"info","ts":"2026-11-03T15:27:43.505Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"987289b5-5f7b-4521-82c0-e2c23fb17fde","1 head service found":"delete-head-after-submit-gg2wh-head-svc"}
{"level":"info","ts":"2026-11-03T15:27:43.505Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"987289b5-5f7b-4521-82c0-e2c23fb17fde","Found 1 head Pod":"delete-head-after-submit-gg2wh-head-b99bx","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:27:43.505Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"987289b5-5f7b-4521-82c0-e2c23fb17fde","head Pod":"delete-head-after-submit-gg2wh-head-b99bx","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod delete-head-after-submit-gg2wh-head-b99bx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:27:43.505Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"987289b5-5f7b-4521-82c0-e2c23fb17fde","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:27:43.505Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"987289b5-5f7b-4521-82c0-e2c23fb17fde","worker Pod":"delete-head-after-submit-gg2wh-small-group-worker-dxpfb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod delete-head-after-submit-gg2wh-small-group-worker-dxpfb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:27:43.505Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"987289b5-5f7b-4521-82c0-e2c23fb17fde","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:27:43.505Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"987289b5-5f7b-4521-82c0-e2c23fb17fde","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:27:43.505Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"987289b5-5f7b-4521-82c0-e2c23fb17fde","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:27:43.505Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"987289b5-5f7b-4521-82c0-e2c23fb17fde","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:27:43.505Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"987289b5-5f7b-4521-82c0-e2c23fb17fde","podName":"delete-head-after-submit-gg2wh-small-group-worker-dxpfb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:27:43.505Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"987289b5-5f7b-4521-82c0-e2c23fb17fde","podName":"delete-head-after-submit-gg2wh-small-group-worker-dxpfb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:27:43.505Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"987289b5-5f7b-4521-82c0-e2c23fb17fde","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:27:43.505Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"987289b5-5f7b-4521-82c0-e2c23fb17fde","seconds":300}
{"level":"info","ts":"2026-11-03T15:27:43.551Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"7844a18d-79f8-4dd2-b008-f3ed4c9a58e5","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:27:43.551Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"7844a18d-79f8-4dd2-b008-f3ed4c9a58e5","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:27:43.551Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"7844a18d-79f8-4dd2-b008-f3ed4c9a58e5","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:27:43.551Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"7844a18d-79f8-4dd2-b008-f3ed4c9a58e5"}
{"level":"info","ts":"2026-11-03T15:27:43.551Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"7844a18d-79f8-4dd2-b008-f3ed4c9a58e5","RayCluster":"delete-head-after-submit-gg2wh","State":""}
{"level":"info","ts":"2026-11-03T15:27:43.551Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"7844a18d-79f8-4dd2-b008-f3ed4c9a58e5","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:43Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:43Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:27:43.559Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"33f5b023-fb03-4ecf-9dba-7c02e3eb14f6"}
{"level":"info","ts":"2026-11-03T15:27:43.559Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"33f5b023-fb03-4ecf-9dba-7c02e3eb14f6"}
{"level":"info","ts":"2026-11-03T15:27:43.559Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"33f5b023-fb03-4ecf-9dba-7c02e3eb14f6","1 head service found":"delete-head-after-submit-gg2wh-head-svc"}
{"level":"info","ts":"2026-11-03T15:27:43.560Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"33f5b023-fb03-4ecf-9dba-7c02e3eb14f6","Found 1 head Pod":"delete-head-after-submit-gg2wh-head-b99bx","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:27:43.560Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"33f5b023-fb03-4ecf-9dba-7c02e3eb14f6","head Pod":"delete-head-after-submit-gg2wh-head-b99bx","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod delete-head-after-submit-gg2wh-head-b99bx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:27:43.560Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"33f5b023-fb03-4ecf-9dba-7c02e3eb14f6","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:27:43.560Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"33f5b023-fb03-4ecf-9dba-7c02e3eb14f6","worker Pod":"delete-head-after-submit-gg2wh-small-group-worker-dxpfb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod delete-head-after-submit-gg2wh-small-group-worker-dxpfb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:27:43.560Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"33f5b023-fb03-4ecf-9dba-7c02e3eb14f6","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:27:43.560Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"33f5b023-fb03-4ecf-9dba-7c02e3eb14f6","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:27:43.560Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"33f5b023-fb03-4ecf-9dba-7c02e3eb14f6","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:27:43.560Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"33f5b023-fb03-4ecf-9dba-7c02e3eb14f6","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:27:43.560Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"33f5b023-fb03-4ecf-9dba-7c02e3eb14f6","podName":"delete-head-after-submit-gg2wh-head-b99bx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:27:43.560Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"33f5b023-fb03-4ecf-9dba-7c02e3eb14f6","podName":"delete-head-after-submit-gg2wh-head-b99bx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:27:43.560Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"33f5b023-fb03-4ecf-9dba-7c02e3eb14f6","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:27:43.560Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"33f5b023-fb03-4ecf-9dba-7c02e3eb14f6","seconds":300}
{"level":"info","ts":"2026-11-03T15:27:43.740Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"11c3b0fe-8d89-4bda-93fd-2a0918bf537b"}
{"level":"info","ts":"2026-11-03T15:27:43.740Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"11c3b0fe-8d89-4bda-93fd-2a0918bf537b"}
{"level":"info","ts":"2026-11-03T15:27:43.740Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"11c3b0fe-8d89-4bda-93fd-2a0918bf537b","1 head service found":"long-running-hvml9-head-svc"}
{"level":"info","ts":"2026-11-03T15:27:43.741Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"11c3b0fe-8d89-4bda-93fd-2a0918bf537b","Found 1 head Pod":"long-running-hvml9-head-s8j5v","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:27:43.741Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"11c3b0fe-8d89-4bda-93fd-2a0918bf537b","head Pod":"long-running-hvml9-head-s8j5v","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-hvml9-head-s8j5v. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:27:43.741Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"11c3b0fe-8d89-4bda-93fd-2a0918bf537b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:27:43.741Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"11c3b0fe-8d89-4bda-93fd-2a0918bf537b","worker Pod":"long-running-hvml9-small-group-worker-mjl4n","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-hvml9-small-group-worker-mjl4n. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:27:43.741Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"11c3b0fe-8d89-4bda-93fd-2a0918bf537b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:27:43.741Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"11c3b0fe-8d89-4bda-93fd-2a0918bf537b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:27:43.741Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"11c3b0fe-8d89-4bda-93fd-2a0918bf537b","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:27:43.741Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"11c3b0fe-8d89-4bda-93fd-2a0918bf537b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:27:43.741Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"11c3b0fe-8d89-4bda-93fd-2a0918bf537b","podName":"long-running-hvml9-small-group-worker-mjl4n","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:27:43.741Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"11c3b0fe-8d89-4bda-93fd-2a0918bf537b","podName":"long-running-hvml9-small-group-worker-mjl4n","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:27:43.741Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"11c3b0fe-8d89-4bda-93fd-2a0918bf537b","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:27:43.741Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"11c3b0fe-8d89-4bda-93fd-2a0918bf537b","seconds":300}
{"level":"info","ts":"2026-11-03T15:27:43.824Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"6da217ec-6131-43ac-8c5c-326f74a18324"}
{"level":"info","ts":"2026-11-03T15:27:43.824Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"6da217ec-6131-43ac-8c5c-326f74a18324"}
{"level":"info","ts":"2026-11-03T15:27:43.824Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"6da217ec-6131-43ac-8c5c-326f74a18324","1 head service found":"long-running-hvml9-head-svc"}
{"level":"info","ts":"2026-11-03T15:27:43.824Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"6da217ec-6131-43ac-8c5c-326f74a18324","Found 1 head Pod":"long-running-hvml9-head-s8j5v","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:27:43.824Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"6da217ec-6131-43ac-8c5c-326f74a18324","head Pod":"long-running-hvml9-head-s8j5v","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-hvml9-head-s8j5v. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:27:43.824Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"6da217ec-6131-43ac-8c5c-326f74a18324","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:27:43.824Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"6da217ec-6131-43ac-8c5c-326f74a18324","worker Pod":"long-running-hvml9-small-group-worker-mjl4n","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-hvml9-small-group-worker-mjl4n. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:27:43.824Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"6da217ec-6131-43ac-8c5c-326f74a18324","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:27:43.824Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"6da217ec-6131-43ac-8c5c-326f74a18324","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:27:43.824Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"6da217ec-6131-43ac-8c5c-326f74a18324","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:27:43.824Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"6da217ec-6131-43ac-8c5c-326f74a18324","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:27:43.825Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"6da217ec-6131-43ac-8c5c-326f74a18324","podName":"long-running-hvml9-head-s8j5v","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:27:43.825Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"6da217ec-6131-43ac-8c5c-326f74a18324","podName":"long-running-hvml9-head-s8j5v","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:27:43.825Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"6da217ec-6131-43ac-8c5c-326f74a18324","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:27:43.825Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"6da217ec-6131-43ac-8c5c-326f74a18324","seconds":300}
{"level":"info","ts":"2026-11-03T15:27:43.869Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"f8186231-4993-4ab7-882e-068c9386c0db","cluster":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:27:43.962Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"ac65cf70-23bd-4538-8007-3c8abdddb696"}
{"level":"info","ts":"2026-11-03T15:27:43.962Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"ac65cf70-23bd-4538-8007-3c8abdddb696"}
{"level":"info","ts":"2026-11-03T15:27:43.962Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"ac65cf70-23bd-4538-8007-3c8abdddb696","1 head service found":"delete-head-after-submit-gg2wh-head-svc"}
{"level":"info","ts":"2026-11-03T15:27:43.963Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"ac65cf70-23bd-4538-8007-3c8abdddb696","Found 1 head Pod":"delete-head-after-submit-gg2wh-head-b99bx","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:27:43.963Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"ac65cf70-23bd-4538-8007-3c8abdddb696","head Pod":"delete-head-after-submit-gg2wh-head-b99bx","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod delete-head-after-submit-gg2wh-head-b99bx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:27:43.963Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"ac65cf70-23bd-4538-8007-3c8abdddb696","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:27:43.963Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"ac65cf70-23bd-4538-8007-3c8abdddb696","worker Pod":"delete-head-after-submit-gg2wh-small-group-worker-dxpfb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod delete-head-after-submit-gg2wh-small-group-worker-dxpfb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:27:43.963Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"ac65cf70-23bd-4538-8007-3c8abdddb696","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:27:43.963Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"ac65cf70-23bd-4538-8007-3c8abdddb696","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:27:43.963Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"ac65cf70-23bd-4538-8007-3c8abdddb696","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:27:43.963Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"ac65cf70-23bd-4538-8007-3c8abdddb696","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:27:43.963Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"ac65cf70-23bd-4538-8007-3c8abdddb696","podName":"delete-head-after-submit-gg2wh-head-b99bx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:27:43.963Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"ac65cf70-23bd-4538-8007-3c8abdddb696","podName":"delete-head-after-submit-gg2wh-head-b99bx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:27:43.963Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"ac65cf70-23bd-4538-8007-3c8abdddb696","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:27:43.963Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"ac65cf70-23bd-4538-8007-3c8abdddb696","seconds":300}
{"level":"info","ts":"2026-11-03T15:27:44.104Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"83a1a8c6-5d00-4f44-a791-f15ee1360009"}
{"level":"info","ts":"2026-11-03T15:27:44.104Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"83a1a8c6-5d00-4f44-a791-f15ee1360009"}
{"level":"info","ts":"2026-11-03T15:27:44.104Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"83a1a8c6-5d00-4f44-a791-f15ee1360009","1 head service found":"long-running-hvml9-head-svc"}
{"level":"info","ts":"2026-11-03T15:27:44.104Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"83a1a8c6-5d00-4f44-a791-f15ee1360009","Found 1 head Pod":"long-running-hvml9-head-s8j5v","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:27:44.104Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"83a1a8c6-5d00-4f44-a791-f15ee1360009","head Pod":"long-running-hvml9-head-s8j5v","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-hvml9-head-s8j5v. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:27:44.104Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"83a1a8c6-5d00-4f44-a791-f15ee1360009","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:27:44.104Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"83a1a8c6-5d00-4f44-a791-f15ee1360009","worker Pod":"long-running-hvml9-small-group-worker-mjl4n","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-hvml9-small-group-worker-mjl4n. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:27:44.104Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"83a1a8c6-5d00-4f44-a791-f15ee1360009","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:27:44.104Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"83a1a8c6-5d00-4f44-a791-f15ee1360009","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:27:44.104Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"83a1a8c6-5d00-4f44-a791-f15ee1360009","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:27:44.104Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"83a1a8c6-5d00-4f44-a791-f15ee1360009","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:27:44.104Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"83a1a8c6-5d00-4f44-a791-f15ee1360009","podName":"long-running-hvml9-head-s8j5v","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:27:44.104Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"83a1a8c6-5d00-4f44-a791-f15ee1360009","podName":"long-running-hvml9-head-s8j5v","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:27:44.104Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"83a1a8c6-5d00-4f44-a791-f15ee1360009","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:27:44.104Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"83a1a8c6-5d00-4f44-a791-f15ee1360009","seconds":300}
{"level":"info","ts":"2026-11-03T15:27:44.144Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"2d4a6337-776d-4930-82d6-3c836d9fe580"}
{"level":"info","ts":"2026-11-03T15:27:44.144Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"2d4a6337-776d-4930-82d6-3c836d9fe580"}
{"level":"info","ts":"2026-11-03T15:27:44.144Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"2d4a6337-776d-4930-82d6-3c836d9fe580","1 head service found":"delete-head-after-submit-gg2wh-head-svc"}
{"level":"info","ts":"2026-11-03T15:27:44.144Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"2d4a6337-776d-4930-82d6-3c836d9fe580","Found 1 head Pod":"delete-head-after-submit-gg2wh-head-b99bx","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:27:44.144Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"2d4a6337-776d-4930-82d6-3c836d9fe580","head Pod":"delete-head-after-submit-gg2wh-head-b99bx","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod delete-head-after-submit-gg2wh-head-b99bx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:27:44.144Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"2d4a6337-776d-4930-82d6-3c836d9fe580","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:27:44.144Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"2d4a6337-776d-4930-82d6-3c836d9fe580","worker Pod":"delete-head-after-submit-gg2wh-small-group-worker-dxpfb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod delete-head-after-submit-gg2wh-small-group-worker-dxpfb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:27:44.144Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"2d4a6337-776d-4930-82d6-3c836d9fe580","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:27:44.144Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"2d4a6337-776d-4930-82d6-3c836d9fe580","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:27:44.144Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"2d4a6337-776d-4930-82d6-3c836d9fe580","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:27:44.144Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"2d4a6337-776d-4930-82d6-3c836d9fe580","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:27:44.144Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"2d4a6337-776d-4930-82d6-3c836d9fe580","podName":"delete-head-after-submit-gg2wh-small-group-worker-dxpfb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:27:44.144Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"2d4a6337-776d-4930-82d6-3c836d9fe580","podName":"delete-head-after-submit-gg2wh-small-group-worker-dxpfb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:27:44.144Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"2d4a6337-776d-4930-82d6-3c836d9fe580","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:27:44.144Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"2d4a6337-776d-4930-82d6-3c836d9fe580","seconds":300}
{"level":"info","ts":"2026-11-03T15:27:44.976Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"38c2cf27-af5c-4df8-a7b5-d10239ace2b5"}
{"level":"info","ts":"2026-11-03T15:27:44.976Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"38c2cf27-af5c-4df8-a7b5-d10239ace2b5"}
{"level":"info","ts":"2026-11-03T15:27:44.976Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"38c2cf27-af5c-4df8-a7b5-d10239ace2b5","1 head service found":"delete-head-after-submit-gg2wh-head-svc"}
{"level":"info","ts":"2026-11-03T15:27:44.976Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"38c2cf27-af5c-4df8-a7b5-d10239ace2b5","Found 1 head Pod":"delete-head-after-submit-gg2wh-head-b99bx","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:27:44.976Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"38c2cf27-af5c-4df8-a7b5-d10239ace2b5","head Pod":"delete-head-after-submit-gg2wh-head-b99bx","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod delete-head-after-submit-gg2wh-head-b99bx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:27:44.976Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"38c2cf27-af5c-4df8-a7b5-d10239ace2b5","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:27:44.977Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"38c2cf27-af5c-4df8-a7b5-d10239ace2b5","worker Pod":"delete-head-after-submit-gg2wh-small-group-worker-dxpfb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod delete-head-after-submit-gg2wh-small-group-worker-dxpfb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:27:44.977Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"38c2cf27-af5c-4df8-a7b5-d10239ace2b5","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:27:44.977Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"38c2cf27-af5c-4df8-a7b5-d10239ace2b5","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:27:44.977Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"38c2cf27-af5c-4df8-a7b5-d10239ace2b5","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:27:44.977Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"38c2cf27-af5c-4df8-a7b5-d10239ace2b5","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:27:44.977Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"38c2cf27-af5c-4df8-a7b5-d10239ace2b5","podName":"delete-head-after-submit-gg2wh-head-b99bx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:27:44.977Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"38c2cf27-af5c-4df8-a7b5-d10239ace2b5","podName":"delete-head-after-submit-gg2wh-head-b99bx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:27:44.977Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"38c2cf27-af5c-4df8-a7b5-d10239ace2b5","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:27:44.977Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"38c2cf27-af5c-4df8-a7b5-d10239ace2b5","seconds":300}
{"level":"info","ts":"2026-11-03T15:27:45.098Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"2f3926cc-3d1c-4ee6-960a-74928d28e2c1"}
{"level":"info","ts":"2026-11-03T15:27:45.098Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"2f3926cc-3d1c-4ee6-960a-74928d28e2c1"}
{"level":"info","ts":"2026-11-03T15:27:45.098Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"2f3926cc-3d1c-4ee6-960a-74928d28e2c1","1 head service found":"long-running-hvml9-head-svc"}
{"level":"info","ts":"2026-11-03T15:27:45.099Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"2f3926cc-3d1c-4ee6-960a-74928d28e2c1","Found 1 head Pod":"long-running-hvml9-head-s8j5v","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:27:45.099Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"2f3926cc-3d1c-4ee6-960a-74928d28e2c1","head Pod":"long-running-hvml9-head-s8j5v","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-hvml9-head-s8j5v. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:27:45.099Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"2f3926cc-3d1c-4ee6-960a-74928d28e2c1","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:27:45.099Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"2f3926cc-3d1c-4ee6-960a-74928d28e2c1","worker Pod":"long-running-hvml9-small-group-worker-mjl4n","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-hvml9-small-group-worker-mjl4n. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:27:45.099Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"2f3926cc-3d1c-4ee6-960a-74928d28e2c1","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:27:45.099Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"2f3926cc-3d1c-4ee6-960a-74928d28e2c1","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:27:45.099Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"2f3926cc-3d1c-4ee6-960a-74928d28e2c1","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:27:45.099Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"2f3926cc-3d1c-4ee6-960a-74928d28e2c1","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:27:45.099Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"2f3926cc-3d1c-4ee6-960a-74928d28e2c1","podName":"long-running-hvml9-head-s8j5v","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:27:45.099Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"2f3926cc-3d1c-4ee6-960a-74928d28e2c1","podName":"long-running-hvml9-head-s8j5v","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:27:45.099Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"2f3926cc-3d1c-4ee6-960a-74928d28e2c1","name":"long-running-hvml9","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:42Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"long-running-hvml9-head-s8j5v","serviceName":"long-running-hvml9-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:32Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.42","serviceIP":"10.132.0.42","podName":"long-running-hvml9-head-s8j5v","serviceName":"long-running-hvml9-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:32Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:27:45.160Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"e8d82574-4370-40bb-8d9d-b34eab9e60c0"}
{"level":"info","ts":"2026-11-03T15:27:45.160Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"e8d82574-4370-40bb-8d9d-b34eab9e60c0"}
{"level":"info","ts":"2026-11-03T15:27:45.160Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"e8d82574-4370-40bb-8d9d-b34eab9e60c0","1 head service found":"delete-head-after-submit-gg2wh-head-svc"}
{"level":"info","ts":"2026-11-03T15:27:45.160Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"e8d82574-4370-40bb-8d9d-b34eab9e60c0","Found 1 head Pod":"delete-head-after-submit-gg2wh-head-b99bx","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:27:45.160Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"e8d82574-4370-40bb-8d9d-b34eab9e60c0","head Pod":"delete-head-after-submit-gg2wh-head-b99bx","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod delete-head-after-submit-gg2wh-head-b99bx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:27:45.160Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"e8d82574-4370-40bb-8d9d-b34eab9e60c0","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:27:45.160Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"e8d82574-4370-40bb-8d9d-b34eab9e60c0","worker Pod":"delete-head-after-submit-gg2wh-small-group-worker-dxpfb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod delete-head-after-submit-gg2wh-small-group-worker-dxpfb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:27:45.160Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"e8d82574-4370-40bb-8d9d-b34eab9e60c0","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:27:45.160Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"e8d82574-4370-40bb-8d9d-b34eab9e60c0","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:27:45.160Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"e8d82574-4370-40bb-8d9d-b34eab9e60c0","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:27:45.160Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"e8d82574-4370-40bb-8d9d-b34eab9e60c0","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:27:45.160Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"bb48d896-2c4a-47b7-84b4-198a4e819ac3","cluster":"long-running-hvml9"}
{"level":"info","ts":"2026-11-03T15:27:45.160Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"e8d82574-4370-40bb-8d9d-b34eab9e60c0","podName":"delete-head-after-submit-gg2wh-head-b99bx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:27:45.160Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"e8d82574-4370-40bb-8d9d-b34eab9e60c0","podName":"delete-head-after-submit-gg2wh-head-b99bx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:27:45.160Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"e8d82574-4370-40bb-8d9d-b34eab9e60c0","name":"delete-head-after-submit-gg2wh","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:43Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:27:45.160Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"241daec4-9688-4a1f-94b6-7ec94e741cad","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:27:45.160Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"241daec4-9688-4a1f-94b6-7ec94e741cad","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Failed","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":600,"Status.endTime":"2026-11-03 15:27:38 +0000 UTC","Now":"2026-11-03T15:27:45.160Z","ShutdownTime":"2026-11-03T15:37:38.000Z"}
{"level":"info","ts":"2026-11-03T15:27:45.160Z","logger":"controllers.RayJob","msg":"TTL has not been met for ShutdownAfterJobFinishes. Requeuing.","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"241daec4-9688-4a1f-94b6-7ec94e741cad","deletionMechanism":"ShutdownAfterJobFinishes","shutdownTime":"2026-11-03T15:37:38.000Z","requeueAfter":594.839168328}
{"level":"info","ts":"2026-11-03T15:27:45.221Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"dc79fc4f-a04a-4246-9a21-45dadac38d84","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:27:45.221Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"dc79fc4f-a04a-4246-9a21-45dadac38d84","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:27:45.221Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"dc79fc4f-a04a-4246-9a21-45dadac38d84","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:27:45.221Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"dc79fc4f-a04a-4246-9a21-45dadac38d84"}
{"level":"info","ts":"2026-11-03T15:27:45.221Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"dc79fc4f-a04a-4246-9a21-45dadac38d84","RayCluster":"delete-head-after-submit-gg2wh","State":""}
{"level":"info","ts":"2026-11-03T15:27:45.221Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"dc79fc4f-a04a-4246-9a21-45dadac38d84","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:43Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:27:45.221Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"dc79fc4f-a04a-4246-9a21-45dadac38d84","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:27:45.282Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"4ec444f2-2026-42aa-afea-da48e4e945ff","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:27:45.282Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"4ec444f2-2026-42aa-afea-da48e4e945ff","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:27:45.282Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"4ec444f2-2026-42aa-afea-da48e4e945ff","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:27:45.282Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"4ec444f2-2026-42aa-afea-da48e4e945ff"}
{"level":"info","ts":"2026-11-03T15:27:45.282Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"4ec444f2-2026-42aa-afea-da48e4e945ff","RayCluster":"delete-head-after-submit-gg2wh","State":""}
{"level":"info","ts":"2026-11-03T15:27:45.282Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"4ec444f2-2026-42aa-afea-da48e4e945ff","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:27:45.465Z","logger":"controllers.RayClusterMTLS","msg":"Updating head certificate","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"bb48d896-2c4a-47b7-84b4-198a4e819ac3","certificate":"ray-head-cert-long-running-hvml9","ipAddresses":["10.132.0.42","127.0.0.1"]}
{"level":"info","ts":"2026-11-03T15:27:45.727Z","logger":"controllers.RayClusterMTLS","msg":"Updating worker certificate","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"bb48d896-2c4a-47b7-84b4-198a4e819ac3","certificate":"ray-worker-cert-long-running-hvml9","ipAddresses":["10.134.0.47","127.0.0.1"]}
{"level":"info","ts":"2026-11-03T15:27:45.988Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"bb48d896-2c4a-47b7-84b4-198a4e819ac3","certificate":"ray-head-cert-long-running-hvml9"}
{"level":"info","ts":"2026-11-03T15:27:45.988Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"bb48d896-2c4a-47b7-84b4-198a4e819ac3"}
{"level":"info","ts":"2026-11-03T15:27:45.988Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"247cc53e-ae97-4ad2-82ed-1d8318823d6e","cluster":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:27:46.294Z","logger":"controllers.RayClusterMTLS","msg":"Updating head certificate","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"247cc53e-ae97-4ad2-82ed-1d8318823d6e","certificate":"ray-head-cert-delete-head-after-submit-gg2wh","ipAddresses":["10.134.0.48","127.0.0.1"]}
{"level":"info","ts":"2026-11-03T15:27:46.555Z","logger":"controllers.RayClusterMTLS","msg":"Updating worker certificate","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"247cc53e-ae97-4ad2-82ed-1d8318823d6e","certificate":"ray-worker-cert-delete-head-after-submit-gg2wh","ipAddresses":["10.132.0.43","127.0.0.1"]}
{"level":"info","ts":"2026-11-03T15:27:46.816Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"247cc53e-ae97-4ad2-82ed-1d8318823d6e","certificate":"ray-head-cert-delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:27:46.816Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"247cc53e-ae97-4ad2-82ed-1d8318823d6e"}
{"level":"info","ts":"2026-11-03T15:27:47.161Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"fa45518d-9b48-48e3-b487-acf2d2d15bb8"}
{"level":"info","ts":"2026-11-03T15:27:47.161Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"fa45518d-9b48-48e3-b487-acf2d2d15bb8"}
{"level":"info","ts":"2026-11-03T15:27:47.161Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"fa45518d-9b48-48e3-b487-acf2d2d15bb8","1 head service found":"long-running-hvml9-head-svc"}
{"level":"info","ts":"2026-11-03T15:27:47.161Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"fa45518d-9b48-48e3-b487-acf2d2d15bb8","Found 1 head Pod":"long-running-hvml9-head-s8j5v","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:27:47.161Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"fa45518d-9b48-48e3-b487-acf2d2d15bb8","head Pod":"long-running-hvml9-head-s8j5v","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-hvml9-head-s8j5v. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:27:47.161Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"fa45518d-9b48-48e3-b487-acf2d2d15bb8","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:27:47.161Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"fa45518d-9b48-48e3-b487-acf2d2d15bb8","worker Pod":"long-running-hvml9-small-group-worker-mjl4n","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-hvml9-small-group-worker-mjl4n. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:27:47.161Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"fa45518d-9b48-48e3-b487-acf2d2d15bb8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:27:47.161Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"fa45518d-9b48-48e3-b487-acf2d2d15bb8","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:27:47.161Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"fa45518d-9b48-48e3-b487-acf2d2d15bb8","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:27:47.161Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"fa45518d-9b48-48e3-b487-acf2d2d15bb8","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:27:47.161Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"fa45518d-9b48-48e3-b487-acf2d2d15bb8","podName":"long-running-hvml9-small-group-worker-mjl4n","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:27:47.161Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"fa45518d-9b48-48e3-b487-acf2d2d15bb8","podName":"long-running-hvml9-small-group-worker-mjl4n","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:27:47.161Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"fa45518d-9b48-48e3-b487-acf2d2d15bb8","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:27:47.161Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"fa45518d-9b48-48e3-b487-acf2d2d15bb8","seconds":300}
{"level":"info","ts":"2026-11-03T15:27:47.222Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"21b84d26-0a59-4a54-b68a-73bdb116e8ff"}
{"level":"info","ts":"2026-11-03T15:27:47.222Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"21b84d26-0a59-4a54-b68a-73bdb116e8ff"}
{"level":"info","ts":"2026-11-03T15:27:47.222Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"21b84d26-0a59-4a54-b68a-73bdb116e8ff","1 head service found":"delete-head-after-submit-gg2wh-head-svc"}
{"level":"info","ts":"2026-11-03T15:27:47.222Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"21b84d26-0a59-4a54-b68a-73bdb116e8ff","Found 1 head Pod":"delete-head-after-submit-gg2wh-head-b99bx","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:27:47.222Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"21b84d26-0a59-4a54-b68a-73bdb116e8ff","head Pod":"delete-head-after-submit-gg2wh-head-b99bx","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod delete-head-after-submit-gg2wh-head-b99bx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:27:47.222Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"21b84d26-0a59-4a54-b68a-73bdb116e8ff","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:27:47.222Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"21b84d26-0a59-4a54-b68a-73bdb116e8ff","worker Pod":"delete-head-after-submit-gg2wh-small-group-worker-dxpfb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod delete-head-after-submit-gg2wh-small-group-worker-dxpfb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:27:47.222Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"21b84d26-0a59-4a54-b68a-73bdb116e8ff","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:27:47.222Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"21b84d26-0a59-4a54-b68a-73bdb116e8ff","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:27:47.222Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"21b84d26-0a59-4a54-b68a-73bdb116e8ff","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:27:47.222Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"21b84d26-0a59-4a54-b68a-73bdb116e8ff","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:27:47.222Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"21b84d26-0a59-4a54-b68a-73bdb116e8ff","podName":"delete-head-after-submit-gg2wh-head-b99bx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:27:47.222Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"21b84d26-0a59-4a54-b68a-73bdb116e8ff","podName":"delete-head-after-submit-gg2wh-head-b99bx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:27:47.222Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"21b84d26-0a59-4a54-b68a-73bdb116e8ff","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:27:47.222Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"21b84d26-0a59-4a54-b68a-73bdb116e8ff","seconds":300}
{"level":"info","ts":"2026-11-03T15:27:48.283Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"f6e2d3a2-dda3-4d6e-bd89-37483cdebbf8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:27:48.283Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"f6e2d3a2-dda3-4d6e-bd89-37483cdebbf8","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:27:48.283Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"f6e2d3a2-dda3-4d6e-bd89-37483cdebbf8","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:27:48.284Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"f6e2d3a2-dda3-4d6e-bd89-37483cdebbf8"}
{"level":"info","ts":"2026-11-03T15:27:48.284Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"f6e2d3a2-dda3-4d6e-bd89-37483cdebbf8","RayCluster":"delete-head-after-submit-gg2wh","State":""}
{"level":"info","ts":"2026-11-03T15:27:48.284Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"f6e2d3a2-dda3-4d6e-bd89-37483cdebbf8","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:27:51.284Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"29516bac-a10b-4f04-925e-40f72caa4707","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:27:51.284Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"29516bac-a10b-4f04-925e-40f72caa4707","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:27:51.284Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"29516bac-a10b-4f04-925e-40f72caa4707","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:27:51.284Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"29516bac-a10b-4f04-925e-40f72caa4707"}
{"level":"info","ts":"2026-11-03T15:27:51.284Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"29516bac-a10b-4f04-925e-40f72caa4707","RayCluster":"delete-head-after-submit-gg2wh","State":""}
{"level":"info","ts":"2026-11-03T15:27:51.284Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"29516bac-a10b-4f04-925e-40f72caa4707","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:27:54.286Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"2abc03ca-d024-4632-92cb-1bd26a1044a3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:27:54.286Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"2abc03ca-d024-4632-92cb-1bd26a1044a3","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:27:54.286Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"2abc03ca-d024-4632-92cb-1bd26a1044a3","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:27:54.286Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"2abc03ca-d024-4632-92cb-1bd26a1044a3"}
{"level":"info","ts":"2026-11-03T15:27:54.286Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"2abc03ca-d024-4632-92cb-1bd26a1044a3","RayCluster":"delete-head-after-submit-gg2wh","State":""}
{"level":"info","ts":"2026-11-03T15:27:54.286Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"2abc03ca-d024-4632-92cb-1bd26a1044a3","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:27:57.287Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"abfbd293-150d-4309-9510-6f976bdfb622","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:27:57.287Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"abfbd293-150d-4309-9510-6f976bdfb622","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:27:57.287Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"abfbd293-150d-4309-9510-6f976bdfb622","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:27:57.287Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"abfbd293-150d-4309-9510-6f976bdfb622"}
{"level":"info","ts":"2026-11-03T15:27:57.287Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"abfbd293-150d-4309-9510-6f976bdfb622","RayCluster":"delete-head-after-submit-gg2wh","State":""}
{"level":"info","ts":"2026-11-03T15:27:57.287Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"abfbd293-150d-4309-9510-6f976bdfb622","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:28:00.287Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"8a36f387-1044-46df-b64c-620d0d3c8557","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:28:00.287Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"8a36f387-1044-46df-b64c-620d0d3c8557","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:28:00.287Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"8a36f387-1044-46df-b64c-620d0d3c8557","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:28:00.288Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"8a36f387-1044-46df-b64c-620d0d3c8557"}
{"level":"info","ts":"2026-11-03T15:28:00.288Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"8a36f387-1044-46df-b64c-620d0d3c8557","RayCluster":"delete-head-after-submit-gg2wh","State":""}
{"level":"info","ts":"2026-11-03T15:28:00.288Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"8a36f387-1044-46df-b64c-620d0d3c8557","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:28:03.288Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"8057aca9-4673-4d96-b2ad-c6c9152410a9","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:28:03.288Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"8057aca9-4673-4d96-b2ad-c6c9152410a9","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:28:03.289Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"8057aca9-4673-4d96-b2ad-c6c9152410a9","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:28:03.289Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"8057aca9-4673-4d96-b2ad-c6c9152410a9"}
{"level":"info","ts":"2026-11-03T15:28:03.289Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"8057aca9-4673-4d96-b2ad-c6c9152410a9","RayCluster":"delete-head-after-submit-gg2wh","State":""}
{"level":"info","ts":"2026-11-03T15:28:03.289Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"8057aca9-4673-4d96-b2ad-c6c9152410a9","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:28:06.289Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"8edd2629-8bc9-4dc2-a723-908db4baa2d5","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:28:06.290Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"8edd2629-8bc9-4dc2-a723-908db4baa2d5","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:28:06.290Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"8edd2629-8bc9-4dc2-a723-908db4baa2d5","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:28:06.290Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"8edd2629-8bc9-4dc2-a723-908db4baa2d5"}
{"level":"info","ts":"2026-11-03T15:28:06.290Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"8edd2629-8bc9-4dc2-a723-908db4baa2d5","RayCluster":"delete-head-after-submit-gg2wh","State":""}
{"level":"info","ts":"2026-11-03T15:28:06.290Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"8edd2629-8bc9-4dc2-a723-908db4baa2d5","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:28:09.291Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"f3fe4acc-e6e8-47ed-aff3-56389cae5e00","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:28:09.291Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"f3fe4acc-e6e8-47ed-aff3-56389cae5e00","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:28:09.291Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"f3fe4acc-e6e8-47ed-aff3-56389cae5e00","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:28:09.291Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"f3fe4acc-e6e8-47ed-aff3-56389cae5e00"}
{"level":"info","ts":"2026-11-03T15:28:09.291Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"f3fe4acc-e6e8-47ed-aff3-56389cae5e00","RayCluster":"delete-head-after-submit-gg2wh","State":""}
{"level":"info","ts":"2026-11-03T15:28:09.291Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"f3fe4acc-e6e8-47ed-aff3-56389cae5e00","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:28:12.292Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"98dfa19d-a0a4-4bb3-bc60-e26ddd2b72cf","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:28:12.292Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"98dfa19d-a0a4-4bb3-bc60-e26ddd2b72cf","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:28:12.292Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"98dfa19d-a0a4-4bb3-bc60-e26ddd2b72cf","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:28:12.292Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"98dfa19d-a0a4-4bb3-bc60-e26ddd2b72cf"}
{"level":"info","ts":"2026-11-03T15:28:12.292Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"98dfa19d-a0a4-4bb3-bc60-e26ddd2b72cf","RayCluster":"delete-head-after-submit-gg2wh","State":""}
{"level":"info","ts":"2026-11-03T15:28:12.292Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"98dfa19d-a0a4-4bb3-bc60-e26ddd2b72cf","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:28:15.293Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"18dff648-55f7-44de-8048-2a3ed38b63da","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:28:15.293Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"18dff648-55f7-44de-8048-2a3ed38b63da","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:28:15.293Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"18dff648-55f7-44de-8048-2a3ed38b63da","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:28:15.293Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"18dff648-55f7-44de-8048-2a3ed38b63da"}
{"level":"info","ts":"2026-11-03T15:28:15.293Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"18dff648-55f7-44de-8048-2a3ed38b63da","RayCluster":"delete-head-after-submit-gg2wh","State":""}
{"level":"info","ts":"2026-11-03T15:28:15.293Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"18dff648-55f7-44de-8048-2a3ed38b63da","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:28:15.989Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"7e1aff4e-2719-4c27-a17c-ce3be8280cf7","cluster":"long-running-hvml9"}
{"level":"info","ts":"2026-11-03T15:28:16.844Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"c3107443-96bd-4753-ab9f-e1e9eaa9da64","cluster":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:28:18.294Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"001b10c1-ccb3-4839-bbec-06031594757a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:28:18.294Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"001b10c1-ccb3-4839-bbec-06031594757a","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:28:18.294Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"001b10c1-ccb3-4839-bbec-06031594757a","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:28:18.294Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"001b10c1-ccb3-4839-bbec-06031594757a"}
{"level":"info","ts":"2026-11-03T15:28:18.294Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"001b10c1-ccb3-4839-bbec-06031594757a","RayCluster":"delete-head-after-submit-gg2wh","State":""}
{"level":"info","ts":"2026-11-03T15:28:18.294Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"001b10c1-ccb3-4839-bbec-06031594757a","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:28:21.294Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"bbad741a-303f-40b9-b79a-80f7177e9e22","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:28:21.294Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"bbad741a-303f-40b9-b79a-80f7177e9e22","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:28:21.294Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"bbad741a-303f-40b9-b79a-80f7177e9e22","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:28:21.295Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"bbad741a-303f-40b9-b79a-80f7177e9e22"}
{"level":"info","ts":"2026-11-03T15:28:21.295Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"bbad741a-303f-40b9-b79a-80f7177e9e22","RayCluster":"delete-head-after-submit-gg2wh","State":""}
{"level":"info","ts":"2026-11-03T15:28:21.295Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"bbad741a-303f-40b9-b79a-80f7177e9e22","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:28:24.295Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"1b5875ab-4d46-486d-87c4-a130f01466a4","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:28:24.295Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"1b5875ab-4d46-486d-87c4-a130f01466a4","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:28:24.296Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"1b5875ab-4d46-486d-87c4-a130f01466a4","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:28:24.296Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"1b5875ab-4d46-486d-87c4-a130f01466a4"}
{"level":"info","ts":"2026-11-03T15:28:24.296Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"1b5875ab-4d46-486d-87c4-a130f01466a4","RayCluster":"delete-head-after-submit-gg2wh","State":""}
{"level":"info","ts":"2026-11-03T15:28:24.296Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"1b5875ab-4d46-486d-87c4-a130f01466a4","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:28:27.297Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"cf0c61cd-0f97-4c9c-8bbd-7f2e78990e4e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:28:27.297Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"cf0c61cd-0f97-4c9c-8bbd-7f2e78990e4e","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:28:27.297Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"cf0c61cd-0f97-4c9c-8bbd-7f2e78990e4e","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:28:27.297Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"cf0c61cd-0f97-4c9c-8bbd-7f2e78990e4e"}
{"level":"info","ts":"2026-11-03T15:28:27.297Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"cf0c61cd-0f97-4c9c-8bbd-7f2e78990e4e","RayCluster":"delete-head-after-submit-gg2wh","State":""}
{"level":"info","ts":"2026-11-03T15:28:27.297Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"cf0c61cd-0f97-4c9c-8bbd-7f2e78990e4e","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:28:30.297Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"26f7a365-4f14-41ff-ba19-3a92499ec9b1","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:28:30.297Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"26f7a365-4f14-41ff-ba19-3a92499ec9b1","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:28:30.297Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"26f7a365-4f14-41ff-ba19-3a92499ec9b1","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:28:30.297Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"26f7a365-4f14-41ff-ba19-3a92499ec9b1"}
{"level":"info","ts":"2026-11-03T15:28:30.297Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"26f7a365-4f14-41ff-ba19-3a92499ec9b1","RayCluster":"delete-head-after-submit-gg2wh","State":""}
{"level":"info","ts":"2026-11-03T15:28:30.297Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"26f7a365-4f14-41ff-ba19-3a92499ec9b1","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:28:33.298Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"7b73cd1e-7d71-4c7f-81fe-312aa88b79a4","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:28:33.298Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"7b73cd1e-7d71-4c7f-81fe-312aa88b79a4","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:28:33.298Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"7b73cd1e-7d71-4c7f-81fe-312aa88b79a4","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:28:33.299Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"7b73cd1e-7d71-4c7f-81fe-312aa88b79a4"}
{"level":"info","ts":"2026-11-03T15:28:33.299Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"7b73cd1e-7d71-4c7f-81fe-312aa88b79a4","RayCluster":"delete-head-after-submit-gg2wh","State":""}
{"level":"info","ts":"2026-11-03T15:28:33.299Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"7b73cd1e-7d71-4c7f-81fe-312aa88b79a4","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:28:36.299Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"9ba264bc-2479-4dee-a10a-bd03b62f2a2d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:28:36.299Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"9ba264bc-2479-4dee-a10a-bd03b62f2a2d","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:28:36.299Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"9ba264bc-2479-4dee-a10a-bd03b62f2a2d","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:28:36.300Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"9ba264bc-2479-4dee-a10a-bd03b62f2a2d"}
{"level":"info","ts":"2026-11-03T15:28:36.300Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"9ba264bc-2479-4dee-a10a-bd03b62f2a2d","RayCluster":"delete-head-after-submit-gg2wh","State":""}
{"level":"info","ts":"2026-11-03T15:28:36.300Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"9ba264bc-2479-4dee-a10a-bd03b62f2a2d","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:28:39.300Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"cbfc2db0-a3e7-43c4-808b-7074c81e18ae","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:28:39.300Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"cbfc2db0-a3e7-43c4-808b-7074c81e18ae","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:28:39.300Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"cbfc2db0-a3e7-43c4-808b-7074c81e18ae","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:28:39.300Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"cbfc2db0-a3e7-43c4-808b-7074c81e18ae"}
{"level":"info","ts":"2026-11-03T15:28:39.300Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"cbfc2db0-a3e7-43c4-808b-7074c81e18ae","RayCluster":"delete-head-after-submit-gg2wh","State":""}
{"level":"info","ts":"2026-11-03T15:28:39.300Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"cbfc2db0-a3e7-43c4-808b-7074c81e18ae","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:28:42.301Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"9e0c4a07-4c2c-4110-8771-1f25c897220f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:28:42.301Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"9e0c4a07-4c2c-4110-8771-1f25c897220f","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:28:42.301Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"9e0c4a07-4c2c-4110-8771-1f25c897220f","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:28:42.301Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"9e0c4a07-4c2c-4110-8771-1f25c897220f"}
{"level":"info","ts":"2026-11-03T15:28:42.301Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"9e0c4a07-4c2c-4110-8771-1f25c897220f","RayCluster":"delete-head-after-submit-gg2wh","State":""}
{"level":"info","ts":"2026-11-03T15:28:42.301Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"9e0c4a07-4c2c-4110-8771-1f25c897220f","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:28:45.302Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"0d6a7cf9-e14c-4910-8e7c-31b3e13c0a20","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:28:45.302Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"0d6a7cf9-e14c-4910-8e7c-31b3e13c0a20","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:28:45.302Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"0d6a7cf9-e14c-4910-8e7c-31b3e13c0a20","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:28:45.302Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"0d6a7cf9-e14c-4910-8e7c-31b3e13c0a20"}
{"level":"info","ts":"2026-11-03T15:28:45.302Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"0d6a7cf9-e14c-4910-8e7c-31b3e13c0a20","RayCluster":"delete-head-after-submit-gg2wh","State":""}
{"level":"info","ts":"2026-11-03T15:28:45.302Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"0d6a7cf9-e14c-4910-8e7c-31b3e13c0a20","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:28:48.303Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"e5e988c5-f5df-47d2-a981-0edd20d10290","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:28:48.303Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"e5e988c5-f5df-47d2-a981-0edd20d10290","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:28:48.303Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"e5e988c5-f5df-47d2-a981-0edd20d10290","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:28:48.303Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"e5e988c5-f5df-47d2-a981-0edd20d10290"}
{"level":"info","ts":"2026-11-03T15:28:48.303Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"e5e988c5-f5df-47d2-a981-0edd20d10290","RayCluster":"delete-head-after-submit-gg2wh","State":""}
{"level":"info","ts":"2026-11-03T15:28:48.303Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"e5e988c5-f5df-47d2-a981-0edd20d10290","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:28:49.415Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"a11cf68d-8607-40c2-940c-c79b0528304d"}
{"level":"info","ts":"2026-11-03T15:28:49.415Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"a11cf68d-8607-40c2-940c-c79b0528304d"}
{"level":"info","ts":"2026-11-03T15:28:49.415Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"a11cf68d-8607-40c2-940c-c79b0528304d","1 head service found":"delete-head-after-submit-gg2wh-head-svc"}
{"level":"info","ts":"2026-11-03T15:28:49.416Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"a11cf68d-8607-40c2-940c-c79b0528304d","Found 1 head Pod":"delete-head-after-submit-gg2wh-head-b99bx","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:28:49.416Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"a11cf68d-8607-40c2-940c-c79b0528304d","head Pod":"delete-head-after-submit-gg2wh-head-b99bx","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod delete-head-after-submit-gg2wh-head-b99bx. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:28:49.416Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"a11cf68d-8607-40c2-940c-c79b0528304d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:28:49.416Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"a11cf68d-8607-40c2-940c-c79b0528304d","worker Pod":"delete-head-after-submit-gg2wh-small-group-worker-dxpfb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod delete-head-after-submit-gg2wh-small-group-worker-dxpfb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:28:49.416Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"a11cf68d-8607-40c2-940c-c79b0528304d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:28:49.416Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"a11cf68d-8607-40c2-940c-c79b0528304d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:28:49.416Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"a11cf68d-8607-40c2-940c-c79b0528304d","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:28:49.416Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"a11cf68d-8607-40c2-940c-c79b0528304d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:28:49.416Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"a11cf68d-8607-40c2-940c-c79b0528304d","podName":"delete-head-after-submit-gg2wh-head-b99bx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:28:49.416Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"a11cf68d-8607-40c2-940c-c79b0528304d","podName":"delete-head-after-submit-gg2wh-head-b99bx","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:28:49.416Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"a11cf68d-8607-40c2-940c-c79b0528304d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:28:49.416Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"a11cf68d-8607-40c2-940c-c79b0528304d","seconds":300}
{"level":"info","ts":"2026-11-03T15:28:50.485Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"597e3315-5bea-4a3f-9b2f-b58c8c03f8ee"}
{"level":"info","ts":"2026-11-03T15:28:50.485Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"597e3315-5bea-4a3f-9b2f-b58c8c03f8ee"}
{"level":"info","ts":"2026-11-03T15:28:50.485Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"597e3315-5bea-4a3f-9b2f-b58c8c03f8ee","1 head service found":"delete-head-after-submit-gg2wh-head-svc"}
{"level":"info","ts":"2026-11-03T15:28:50.485Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"597e3315-5bea-4a3f-9b2f-b58c8c03f8ee","Found 1 head Pod":"delete-head-after-submit-gg2wh-head-b99bx","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:28:50.485Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"597e3315-5bea-4a3f-9b2f-b58c8c03f8ee","head Pod":"delete-head-after-submit-gg2wh-head-b99bx","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod delete-head-after-submit-gg2wh-head-b99bx. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:28:50.485Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"597e3315-5bea-4a3f-9b2f-b58c8c03f8ee","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:28:50.485Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"597e3315-5bea-4a3f-9b2f-b58c8c03f8ee","worker Pod":"delete-head-after-submit-gg2wh-small-group-worker-dxpfb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod delete-head-after-submit-gg2wh-small-group-worker-dxpfb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:28:50.485Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"597e3315-5bea-4a3f-9b2f-b58c8c03f8ee","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:28:50.485Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"597e3315-5bea-4a3f-9b2f-b58c8c03f8ee","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:28:50.485Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"597e3315-5bea-4a3f-9b2f-b58c8c03f8ee","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:28:50.485Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"597e3315-5bea-4a3f-9b2f-b58c8c03f8ee","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:28:50.486Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"597e3315-5bea-4a3f-9b2f-b58c8c03f8ee","podName":"delete-head-after-submit-gg2wh-small-group-worker-dxpfb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:28:50.486Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"597e3315-5bea-4a3f-9b2f-b58c8c03f8ee","podName":"delete-head-after-submit-gg2wh-small-group-worker-dxpfb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:28:50.486Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"597e3315-5bea-4a3f-9b2f-b58c8c03f8ee","name":"delete-head-after-submit-gg2wh","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:28:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:28:50.547Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"f0f68aaf-c940-4e09-a1ad-a32b8120d37b","cluster":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:28:50.547Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"63513be1-5175-4313-b631-16b7115f5cf2","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:28:50.547Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"63513be1-5175-4313-b631-16b7115f5cf2","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:28:50.547Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"63513be1-5175-4313-b631-16b7115f5cf2","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:28:50.548Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"63513be1-5175-4313-b631-16b7115f5cf2"}
{"level":"info","ts":"2026-11-03T15:28:50.548Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"63513be1-5175-4313-b631-16b7115f5cf2","RayCluster":"delete-head-after-submit-gg2wh","State":""}
{"level":"info","ts":"2026-11-03T15:28:50.548Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"63513be1-5175-4313-b631-16b7115f5cf2","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:28:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:28:50.548Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"63513be1-5175-4313-b631-16b7115f5cf2","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:28:50.609Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"dff281f4-7b97-46bb-b301-917617055155","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:28:50.609Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"dff281f4-7b97-46bb-b301-917617055155","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:28:50.609Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"dff281f4-7b97-46bb-b301-917617055155","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:28:50.609Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"dff281f4-7b97-46bb-b301-917617055155"}
{"level":"info","ts":"2026-11-03T15:28:50.609Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"dff281f4-7b97-46bb-b301-917617055155","RayCluster":"delete-head-after-submit-gg2wh","State":""}
{"level":"info","ts":"2026-11-03T15:28:50.609Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"dff281f4-7b97-46bb-b301-917617055155","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:28:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:28:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:28:52.547Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"13c0f39b-ad19-450a-bb0d-5fc5442a278e"}
{"level":"info","ts":"2026-11-03T15:28:52.547Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"13c0f39b-ad19-450a-bb0d-5fc5442a278e"}
{"level":"info","ts":"2026-11-03T15:28:52.547Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"13c0f39b-ad19-450a-bb0d-5fc5442a278e","1 head service found":"delete-head-after-submit-gg2wh-head-svc"}
{"level":"info","ts":"2026-11-03T15:28:52.547Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"13c0f39b-ad19-450a-bb0d-5fc5442a278e","Found 1 head Pod":"delete-head-after-submit-gg2wh-head-b99bx","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:28:52.547Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"13c0f39b-ad19-450a-bb0d-5fc5442a278e","head Pod":"delete-head-after-submit-gg2wh-head-b99bx","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod delete-head-after-submit-gg2wh-head-b99bx. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:28:52.547Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"13c0f39b-ad19-450a-bb0d-5fc5442a278e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:28:52.547Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"13c0f39b-ad19-450a-bb0d-5fc5442a278e","worker Pod":"delete-head-after-submit-gg2wh-small-group-worker-dxpfb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod delete-head-after-submit-gg2wh-small-group-worker-dxpfb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:28:52.547Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"13c0f39b-ad19-450a-bb0d-5fc5442a278e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:28:52.547Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"13c0f39b-ad19-450a-bb0d-5fc5442a278e","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:28:52.547Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"13c0f39b-ad19-450a-bb0d-5fc5442a278e","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:28:52.547Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"13c0f39b-ad19-450a-bb0d-5fc5442a278e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:28:52.547Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"13c0f39b-ad19-450a-bb0d-5fc5442a278e","podName":"delete-head-after-submit-gg2wh-head-b99bx","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:27:43Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:28:52.548Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"13c0f39b-ad19-450a-bb0d-5fc5442a278e","podName":"delete-head-after-submit-gg2wh-head-b99bx","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:27:43Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:28:52.548Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"13c0f39b-ad19-450a-bb0d-5fc5442a278e","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:28:52.548Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"13c0f39b-ad19-450a-bb0d-5fc5442a278e","seconds":300}
{"level":"info","ts":"2026-11-03T15:28:53.609Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"a5489839-02f9-4422-8f60-288335dfbd72","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:28:53.609Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"a5489839-02f9-4422-8f60-288335dfbd72","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:28:53.609Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"a5489839-02f9-4422-8f60-288335dfbd72","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:28:53.610Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"a5489839-02f9-4422-8f60-288335dfbd72"}
{"level":"info","ts":"2026-11-03T15:28:53.610Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"a5489839-02f9-4422-8f60-288335dfbd72","RayCluster":"delete-head-after-submit-gg2wh","State":""}
{"level":"info","ts":"2026-11-03T15:28:53.610Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"a5489839-02f9-4422-8f60-288335dfbd72","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:28:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:28:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:28:54.345Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"ffc1f603-5ba4-4baf-9866-6f9ee6f42ee8"}
{"level":"info","ts":"2026-11-03T15:28:54.345Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"ffc1f603-5ba4-4baf-9866-6f9ee6f42ee8"}
{"level":"info","ts":"2026-11-03T15:28:54.345Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"ffc1f603-5ba4-4baf-9866-6f9ee6f42ee8","1 head service found":"long-running-hvml9-head-svc"}
{"level":"info","ts":"2026-11-03T15:28:54.345Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"ffc1f603-5ba4-4baf-9866-6f9ee6f42ee8","Found 1 head Pod":"long-running-hvml9-head-s8j5v","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:28:54.346Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"ffc1f603-5ba4-4baf-9866-6f9ee6f42ee8","head Pod":"long-running-hvml9-head-s8j5v","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-hvml9-head-s8j5v. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:28:54.346Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"ffc1f603-5ba4-4baf-9866-6f9ee6f42ee8","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:28:54.346Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"ffc1f603-5ba4-4baf-9866-6f9ee6f42ee8","worker Pod":"long-running-hvml9-small-group-worker-mjl4n","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-hvml9-small-group-worker-mjl4n. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:28:54.346Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"ffc1f603-5ba4-4baf-9866-6f9ee6f42ee8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:28:54.346Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"ffc1f603-5ba4-4baf-9866-6f9ee6f42ee8","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:28:54.346Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"ffc1f603-5ba4-4baf-9866-6f9ee6f42ee8","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:28:54.346Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"ffc1f603-5ba4-4baf-9866-6f9ee6f42ee8","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:28:54.346Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"ffc1f603-5ba4-4baf-9866-6f9ee6f42ee8","podName":"long-running-hvml9-head-s8j5v","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:28:54.346Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"ffc1f603-5ba4-4baf-9866-6f9ee6f42ee8","podName":"long-running-hvml9-head-s8j5v","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:28:54.346Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"ffc1f603-5ba4-4baf-9866-6f9ee6f42ee8","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:28:54.346Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"ffc1f603-5ba4-4baf-9866-6f9ee6f42ee8","seconds":300}
{"level":"info","ts":"2026-11-03T15:28:55.373Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"e9857d21-f877-4cee-93d6-de1eea98de8d"}
{"level":"info","ts":"2026-11-03T15:28:55.373Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"e9857d21-f877-4cee-93d6-de1eea98de8d"}
{"level":"info","ts":"2026-11-03T15:28:55.373Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"e9857d21-f877-4cee-93d6-de1eea98de8d","1 head service found":"long-running-hvml9-head-svc"}
{"level":"info","ts":"2026-11-03T15:28:55.373Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"e9857d21-f877-4cee-93d6-de1eea98de8d","Found 1 head Pod":"long-running-hvml9-head-s8j5v","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:28:55.373Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"e9857d21-f877-4cee-93d6-de1eea98de8d","head Pod":"long-running-hvml9-head-s8j5v","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-hvml9-head-s8j5v. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:28:55.373Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"e9857d21-f877-4cee-93d6-de1eea98de8d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:28:55.374Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"e9857d21-f877-4cee-93d6-de1eea98de8d","worker Pod":"long-running-hvml9-small-group-worker-mjl4n","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-hvml9-small-group-worker-mjl4n. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:28:55.374Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"e9857d21-f877-4cee-93d6-de1eea98de8d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:28:55.374Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"e9857d21-f877-4cee-93d6-de1eea98de8d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:28:55.374Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"e9857d21-f877-4cee-93d6-de1eea98de8d","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:28:55.374Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"e9857d21-f877-4cee-93d6-de1eea98de8d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:28:55.374Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"e9857d21-f877-4cee-93d6-de1eea98de8d","podName":"long-running-hvml9-small-group-worker-mjl4n","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:28:55.374Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"e9857d21-f877-4cee-93d6-de1eea98de8d","podName":"long-running-hvml9-small-group-worker-mjl4n","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:28:55.374Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"e9857d21-f877-4cee-93d6-de1eea98de8d","name":"long-running-hvml9","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:27:45Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.42","serviceIP":"10.132.0.42","podName":"long-running-hvml9-head-s8j5v","serviceName":"long-running-hvml9-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:32Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:28:55Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.42","serviceIP":"10.132.0.42","podName":"long-running-hvml9-head-s8j5v","serviceName":"long-running-hvml9-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:32Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:28:55.436Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"c53b73cb-ad89-4780-8837-099e667abfdd","cluster":"long-running-hvml9"}
{"level":"info","ts":"2026-11-03T15:28:55.437Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"7555407b-0cbe-450b-bd3e-8f2388c8c0ea","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:28:55.437Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"7555407b-0cbe-450b-bd3e-8f2388c8c0ea","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Failed","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":600,"Status.endTime":"2026-11-03 15:27:38 +0000 UTC","Now":"2026-11-03T15:28:55.437Z","ShutdownTime":"2026-11-03T15:37:38.000Z"}
{"level":"info","ts":"2026-11-03T15:28:55.437Z","logger":"controllers.RayJob","msg":"TTL has not been met for ShutdownAfterJobFinishes. Requeuing.","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"7555407b-0cbe-450b-bd3e-8f2388c8c0ea","deletionMechanism":"ShutdownAfterJobFinishes","shutdownTime":"2026-11-03T15:37:38.000Z","requeueAfter":524.562941203}
{"level":"info","ts":"2026-11-03T15:28:56.611Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"73740420-3322-4c8a-94de-6fa33f8fa616","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:28:56.611Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"73740420-3322-4c8a-94de-6fa33f8fa616","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:28:56.611Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"73740420-3322-4c8a-94de-6fa33f8fa616","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:28:56.611Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"73740420-3322-4c8a-94de-6fa33f8fa616"}
{"level":"info","ts":"2026-11-03T15:28:56.611Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"73740420-3322-4c8a-94de-6fa33f8fa616","RayCluster":"delete-head-after-submit-gg2wh","State":""}
{"level":"info","ts":"2026-11-03T15:28:56.611Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"73740420-3322-4c8a-94de-6fa33f8fa616","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:28:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:28:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:28:57.439Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"3144b25d-6917-47ac-bae4-f0338df78223"}
{"level":"info","ts":"2026-11-03T15:28:57.439Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"3144b25d-6917-47ac-bae4-f0338df78223"}
{"level":"info","ts":"2026-11-03T15:28:57.439Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"3144b25d-6917-47ac-bae4-f0338df78223","1 head service found":"long-running-hvml9-head-svc"}
{"level":"info","ts":"2026-11-03T15:28:57.439Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"3144b25d-6917-47ac-bae4-f0338df78223","Found 1 head Pod":"long-running-hvml9-head-s8j5v","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:28:57.439Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"3144b25d-6917-47ac-bae4-f0338df78223","head Pod":"long-running-hvml9-head-s8j5v","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-hvml9-head-s8j5v. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:28:57.439Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"3144b25d-6917-47ac-bae4-f0338df78223","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:28:57.439Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"3144b25d-6917-47ac-bae4-f0338df78223","worker Pod":"long-running-hvml9-small-group-worker-mjl4n","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-hvml9-small-group-worker-mjl4n. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:28:57.439Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"3144b25d-6917-47ac-bae4-f0338df78223","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:28:57.439Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"3144b25d-6917-47ac-bae4-f0338df78223","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:28:57.439Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"3144b25d-6917-47ac-bae4-f0338df78223","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:28:57.439Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"3144b25d-6917-47ac-bae4-f0338df78223","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:28:57.439Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"3144b25d-6917-47ac-bae4-f0338df78223","podName":"long-running-hvml9-head-s8j5v","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:27:42Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:28:57.439Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"3144b25d-6917-47ac-bae4-f0338df78223","podName":"long-running-hvml9-head-s8j5v","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:27:42Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:28:57.439Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"3144b25d-6917-47ac-bae4-f0338df78223","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:28:57.439Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"3144b25d-6917-47ac-bae4-f0338df78223","seconds":300}
{"level":"info","ts":"2026-11-03T15:28:59.612Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"ed36f8b9-ba86-49fb-8e74-450259e87026","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:28:59.612Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"ed36f8b9-ba86-49fb-8e74-450259e87026","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:28:59.612Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"ed36f8b9-ba86-49fb-8e74-450259e87026","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:28:59.612Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"ed36f8b9-ba86-49fb-8e74-450259e87026"}
{"level":"info","ts":"2026-11-03T15:28:59.612Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"ed36f8b9-ba86-49fb-8e74-450259e87026","RayCluster":"delete-head-after-submit-gg2wh","State":""}
{"level":"info","ts":"2026-11-03T15:28:59.612Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"ed36f8b9-ba86-49fb-8e74-450259e87026","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:28:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:28:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:29:01.505Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"bb9bd11c-5fcf-46e3-858d-48e4ba1f4e64"}
{"level":"info","ts":"2026-11-03T15:29:01.505Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"bb9bd11c-5fcf-46e3-858d-48e4ba1f4e64"}
{"level":"info","ts":"2026-11-03T15:29:01.505Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"bb9bd11c-5fcf-46e3-858d-48e4ba1f4e64","1 head service found":"delete-head-after-submit-gg2wh-head-svc"}
{"level":"info","ts":"2026-11-03T15:29:01.505Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"bb9bd11c-5fcf-46e3-858d-48e4ba1f4e64","Found 1 head Pod":"delete-head-after-submit-gg2wh-head-b99bx","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:29:01.505Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"bb9bd11c-5fcf-46e3-858d-48e4ba1f4e64","head Pod":"delete-head-after-submit-gg2wh-head-b99bx","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod delete-head-after-submit-gg2wh-head-b99bx. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:29:01.505Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"bb9bd11c-5fcf-46e3-858d-48e4ba1f4e64","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:29:01.505Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"bb9bd11c-5fcf-46e3-858d-48e4ba1f4e64","worker Pod":"delete-head-after-submit-gg2wh-small-group-worker-dxpfb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod delete-head-after-submit-gg2wh-small-group-worker-dxpfb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:29:01.505Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"bb9bd11c-5fcf-46e3-858d-48e4ba1f4e64","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:29:01.505Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"bb9bd11c-5fcf-46e3-858d-48e4ba1f4e64","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:29:01.505Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"bb9bd11c-5fcf-46e3-858d-48e4ba1f4e64","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:29:01.505Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"bb9bd11c-5fcf-46e3-858d-48e4ba1f4e64","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:29:01.505Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"bb9bd11c-5fcf-46e3-858d-48e4ba1f4e64","podName":"delete-head-after-submit-gg2wh-small-group-worker-dxpfb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:29:01.505Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"bb9bd11c-5fcf-46e3-858d-48e4ba1f4e64","podName":"delete-head-after-submit-gg2wh-small-group-worker-dxpfb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:29:01.505Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"bb9bd11c-5fcf-46e3-858d-48e4ba1f4e64","name":"delete-head-after-submit-gg2wh","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:28:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:29:01Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:29:01Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:29:01.569Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"78174c4a-c9ec-4b1f-9cff-c2a25cbc9838","cluster":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:29:01.569Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"31305e48-9445-4c57-b0cb-ab72e0ecf4e8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:29:01.569Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"31305e48-9445-4c57-b0cb-ab72e0ecf4e8","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:29:01.569Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"31305e48-9445-4c57-b0cb-ab72e0ecf4e8","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:29:01.570Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"31305e48-9445-4c57-b0cb-ab72e0ecf4e8"}
{"level":"info","ts":"2026-11-03T15:29:01.570Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"31305e48-9445-4c57-b0cb-ab72e0ecf4e8","RayCluster":"delete-head-after-submit-gg2wh","State":""}
{"level":"info","ts":"2026-11-03T15:29:01.570Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"31305e48-9445-4c57-b0cb-ab72e0ecf4e8","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:28:50Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:29:01Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:29:01Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:29:01.570Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"31305e48-9445-4c57-b0cb-ab72e0ecf4e8","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:29:01.635Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"742ae81d-f5b7-4892-a852-5fd4cc2853b5","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:29:01.635Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"742ae81d-f5b7-4892-a852-5fd4cc2853b5","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:29:01.635Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"742ae81d-f5b7-4892-a852-5fd4cc2853b5","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:29:01.635Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"742ae81d-f5b7-4892-a852-5fd4cc2853b5"}
{"level":"info","ts":"2026-11-03T15:29:01.635Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"742ae81d-f5b7-4892-a852-5fd4cc2853b5","RayCluster":"delete-head-after-submit-gg2wh","State":""}
{"level":"info","ts":"2026-11-03T15:29:01.635Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"742ae81d-f5b7-4892-a852-5fd4cc2853b5","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:29:01Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:29:01Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:29:01Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:29:01Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:29:03.569Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"36052a86-5b7c-44f5-8c4d-fa57dc94d905"}
{"level":"info","ts":"2026-11-03T15:29:03.569Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"36052a86-5b7c-44f5-8c4d-fa57dc94d905"}
{"level":"info","ts":"2026-11-03T15:29:03.570Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"36052a86-5b7c-44f5-8c4d-fa57dc94d905","1 head service found":"delete-head-after-submit-gg2wh-head-svc"}
{"level":"info","ts":"2026-11-03T15:29:03.570Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"36052a86-5b7c-44f5-8c4d-fa57dc94d905","Found 1 head Pod":"delete-head-after-submit-gg2wh-head-b99bx","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:29:03.570Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"36052a86-5b7c-44f5-8c4d-fa57dc94d905","head Pod":"delete-head-after-submit-gg2wh-head-b99bx","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod delete-head-after-submit-gg2wh-head-b99bx. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:29:03.570Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"36052a86-5b7c-44f5-8c4d-fa57dc94d905","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:29:03.570Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"36052a86-5b7c-44f5-8c4d-fa57dc94d905","worker Pod":"delete-head-after-submit-gg2wh-small-group-worker-dxpfb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod delete-head-after-submit-gg2wh-small-group-worker-dxpfb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:29:03.570Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"36052a86-5b7c-44f5-8c4d-fa57dc94d905","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:29:03.570Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"36052a86-5b7c-44f5-8c4d-fa57dc94d905","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:29:03.570Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"36052a86-5b7c-44f5-8c4d-fa57dc94d905","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:29:03.570Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"36052a86-5b7c-44f5-8c4d-fa57dc94d905","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:29:03.570Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"36052a86-5b7c-44f5-8c4d-fa57dc94d905","podName":"delete-head-after-submit-gg2wh-small-group-worker-dxpfb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:29:03.570Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"36052a86-5b7c-44f5-8c4d-fa57dc94d905","podName":"delete-head-after-submit-gg2wh-small-group-worker-dxpfb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:29:03.570Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"36052a86-5b7c-44f5-8c4d-fa57dc94d905","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:29:03.570Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"36052a86-5b7c-44f5-8c4d-fa57dc94d905","seconds":300}
{"level":"info","ts":"2026-11-03T15:29:04.636Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"877b42cf-d58a-42d2-a6be-00a54306db1e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:29:04.636Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"877b42cf-d58a-42d2-a6be-00a54306db1e","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:29:04.636Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"877b42cf-d58a-42d2-a6be-00a54306db1e","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:29:04.636Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"877b42cf-d58a-42d2-a6be-00a54306db1e"}
{"level":"info","ts":"2026-11-03T15:29:04.636Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"877b42cf-d58a-42d2-a6be-00a54306db1e","RayCluster":"delete-head-after-submit-gg2wh","State":""}
{"level":"info","ts":"2026-11-03T15:29:04.636Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"877b42cf-d58a-42d2-a6be-00a54306db1e","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:29:01Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:29:01Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:29:01Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:29:01Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:29:06.366Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"a73a3d35-d9c4-4542-9299-6692d0ab28c6"}
{"level":"info","ts":"2026-11-03T15:29:06.366Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"a73a3d35-d9c4-4542-9299-6692d0ab28c6"}
{"level":"info","ts":"2026-11-03T15:29:06.366Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"a73a3d35-d9c4-4542-9299-6692d0ab28c6","1 head service found":"long-running-hvml9-head-svc"}
{"level":"info","ts":"2026-11-03T15:29:06.366Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"a73a3d35-d9c4-4542-9299-6692d0ab28c6","Found 1 head Pod":"long-running-hvml9-head-s8j5v","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:29:06.366Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"a73a3d35-d9c4-4542-9299-6692d0ab28c6","head Pod":"long-running-hvml9-head-s8j5v","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-hvml9-head-s8j5v. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:29:06.366Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"a73a3d35-d9c4-4542-9299-6692d0ab28c6","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:29:06.366Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"a73a3d35-d9c4-4542-9299-6692d0ab28c6","worker Pod":"long-running-hvml9-small-group-worker-mjl4n","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-hvml9-small-group-worker-mjl4n. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:29:06.366Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"a73a3d35-d9c4-4542-9299-6692d0ab28c6","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:29:06.366Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"a73a3d35-d9c4-4542-9299-6692d0ab28c6","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:29:06.366Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"a73a3d35-d9c4-4542-9299-6692d0ab28c6","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:29:06.366Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"a73a3d35-d9c4-4542-9299-6692d0ab28c6","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:29:06.366Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"a73a3d35-d9c4-4542-9299-6692d0ab28c6","podName":"long-running-hvml9-small-group-worker-mjl4n","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:29:06.366Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"a73a3d35-d9c4-4542-9299-6692d0ab28c6","podName":"long-running-hvml9-small-group-worker-mjl4n","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:29:06.366Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"a73a3d35-d9c4-4542-9299-6692d0ab28c6","name":"long-running-hvml9","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:28:55Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.42","serviceIP":"10.132.0.42","podName":"long-running-hvml9-head-s8j5v","serviceName":"long-running-hvml9-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:32Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:29:06Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.42","serviceIP":"10.132.0.42","podName":"long-running-hvml9-head-s8j5v","serviceName":"long-running-hvml9-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:29:06Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:32Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:29:06.428Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"aa8f9196-89a4-4779-9c69-6d1def9fe6cb","cluster":"long-running-hvml9"}
{"level":"info","ts":"2026-11-03T15:29:06.428Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"e68e2fa5-7b65-4175-9868-0bed0b22628b","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:29:06.428Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"e68e2fa5-7b65-4175-9868-0bed0b22628b","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Failed","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":600,"Status.endTime":"2026-11-03 15:27:38 +0000 UTC","Now":"2026-11-03T15:29:06.428Z","ShutdownTime":"2026-11-03T15:37:38.000Z"}
{"level":"info","ts":"2026-11-03T15:29:06.428Z","logger":"controllers.RayJob","msg":"TTL has not been met for ShutdownAfterJobFinishes. Requeuing.","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"e68e2fa5-7b65-4175-9868-0bed0b22628b","deletionMechanism":"ShutdownAfterJobFinishes","shutdownTime":"2026-11-03T15:37:38.000Z","requeueAfter":513.571319201}
{"level":"info","ts":"2026-11-03T15:29:07.637Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"18252421-5c47-4fac-b5e6-3608a367b3e4","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:29:07.637Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"18252421-5c47-4fac-b5e6-3608a367b3e4","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:29:07.637Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"18252421-5c47-4fac-b5e6-3608a367b3e4","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:29:07.638Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"18252421-5c47-4fac-b5e6-3608a367b3e4"}
{"level":"info","ts":"2026-11-03T15:29:07.638Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"18252421-5c47-4fac-b5e6-3608a367b3e4","RayCluster":"delete-head-after-submit-gg2wh","State":""}
{"level":"info","ts":"2026-11-03T15:29:07.638Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"18252421-5c47-4fac-b5e6-3608a367b3e4","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:29:01Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:29:01Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:29:01Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:29:01Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:29:08.428Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"6d019db1-9f6a-423b-bcef-2006dd7b0771"}
{"level":"info","ts":"2026-11-03T15:29:08.428Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"6d019db1-9f6a-423b-bcef-2006dd7b0771"}
{"level":"info","ts":"2026-11-03T15:29:08.428Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"6d019db1-9f6a-423b-bcef-2006dd7b0771","1 head service found":"long-running-hvml9-head-svc"}
{"level":"info","ts":"2026-11-03T15:29:08.428Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"6d019db1-9f6a-423b-bcef-2006dd7b0771","Found 1 head Pod":"long-running-hvml9-head-s8j5v","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:29:08.428Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"6d019db1-9f6a-423b-bcef-2006dd7b0771","head Pod":"long-running-hvml9-head-s8j5v","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-hvml9-head-s8j5v. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:29:08.428Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"6d019db1-9f6a-423b-bcef-2006dd7b0771","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:29:08.429Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"6d019db1-9f6a-423b-bcef-2006dd7b0771","worker Pod":"long-running-hvml9-small-group-worker-mjl4n","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-hvml9-small-group-worker-mjl4n. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:29:08.429Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"6d019db1-9f6a-423b-bcef-2006dd7b0771","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:29:08.429Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"6d019db1-9f6a-423b-bcef-2006dd7b0771","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:29:08.429Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"6d019db1-9f6a-423b-bcef-2006dd7b0771","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:29:08.429Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"6d019db1-9f6a-423b-bcef-2006dd7b0771","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:29:08.429Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"6d019db1-9f6a-423b-bcef-2006dd7b0771","podName":"long-running-hvml9-small-group-worker-mjl4n","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:29:08.429Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"6d019db1-9f6a-423b-bcef-2006dd7b0771","podName":"long-running-hvml9-small-group-worker-mjl4n","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:29:08.429Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"6d019db1-9f6a-423b-bcef-2006dd7b0771","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:29:08.429Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"6d019db1-9f6a-423b-bcef-2006dd7b0771","seconds":300}
{"level":"info","ts":"2026-11-03T15:29:10.639Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"ce8a9973-686f-4830-90a4-485e23b5aa3e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:29:10.639Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"ce8a9973-686f-4830-90a4-485e23b5aa3e","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:29:10.639Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"ce8a9973-686f-4830-90a4-485e23b5aa3e","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:29:10.639Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"ce8a9973-686f-4830-90a4-485e23b5aa3e"}
{"level":"info","ts":"2026-11-03T15:29:10.639Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"ce8a9973-686f-4830-90a4-485e23b5aa3e","RayCluster":"delete-head-after-submit-gg2wh","State":""}
{"level":"info","ts":"2026-11-03T15:29:10.639Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"ce8a9973-686f-4830-90a4-485e23b5aa3e","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:29:01Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:29:01Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:29:01Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:29:01Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:29:11.420Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"ec6d4a7f-7044-4804-985b-23488ae9a9f7"}
{"level":"info","ts":"2026-11-03T15:29:11.420Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"ec6d4a7f-7044-4804-985b-23488ae9a9f7"}
{"level":"info","ts":"2026-11-03T15:29:11.420Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"ec6d4a7f-7044-4804-985b-23488ae9a9f7","1 head service found":"delete-head-after-submit-gg2wh-head-svc"}
{"level":"info","ts":"2026-11-03T15:29:11.420Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"ec6d4a7f-7044-4804-985b-23488ae9a9f7","Found 1 head Pod":"delete-head-after-submit-gg2wh-head-b99bx","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:29:11.420Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"ec6d4a7f-7044-4804-985b-23488ae9a9f7","head Pod":"delete-head-after-submit-gg2wh-head-b99bx","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod delete-head-after-submit-gg2wh-head-b99bx. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:29:11.420Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"ec6d4a7f-7044-4804-985b-23488ae9a9f7","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:29:11.420Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"ec6d4a7f-7044-4804-985b-23488ae9a9f7","worker Pod":"delete-head-after-submit-gg2wh-small-group-worker-dxpfb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod delete-head-after-submit-gg2wh-small-group-worker-dxpfb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:29:11.420Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"ec6d4a7f-7044-4804-985b-23488ae9a9f7","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:29:11.420Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"ec6d4a7f-7044-4804-985b-23488ae9a9f7","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:29:11.420Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"ec6d4a7f-7044-4804-985b-23488ae9a9f7","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:29:11.420Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"ec6d4a7f-7044-4804-985b-23488ae9a9f7","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:29:11.420Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"ec6d4a7f-7044-4804-985b-23488ae9a9f7","podName":"delete-head-after-submit-gg2wh-small-group-worker-dxpfb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:29:11.420Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"ec6d4a7f-7044-4804-985b-23488ae9a9f7","podName":"delete-head-after-submit-gg2wh-small-group-worker-dxpfb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:29:11.420Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"ec6d4a7f-7044-4804-985b-23488ae9a9f7","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:29:11.420Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"ec6d4a7f-7044-4804-985b-23488ae9a9f7","seconds":300}
{"level":"info","ts":"2026-11-03T15:29:12.378Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"083c4da6-8c2b-442a-a286-93d5ac659bca"}
{"level":"info","ts":"2026-11-03T15:29:12.378Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"083c4da6-8c2b-442a-a286-93d5ac659bca"}
{"level":"info","ts":"2026-11-03T15:29:12.378Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"083c4da6-8c2b-442a-a286-93d5ac659bca","1 head service found":"delete-head-after-submit-gg2wh-head-svc"}
{"level":"info","ts":"2026-11-03T15:29:12.379Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"083c4da6-8c2b-442a-a286-93d5ac659bca","Found 1 head Pod":"delete-head-after-submit-gg2wh-head-b99bx","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:29:12.379Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"083c4da6-8c2b-442a-a286-93d5ac659bca","head Pod":"delete-head-after-submit-gg2wh-head-b99bx","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod delete-head-after-submit-gg2wh-head-b99bx. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:29:12.379Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"083c4da6-8c2b-442a-a286-93d5ac659bca","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:29:12.379Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"083c4da6-8c2b-442a-a286-93d5ac659bca","worker Pod":"delete-head-after-submit-gg2wh-small-group-worker-dxpfb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod delete-head-after-submit-gg2wh-small-group-worker-dxpfb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:29:12.379Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"083c4da6-8c2b-442a-a286-93d5ac659bca","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:29:12.379Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"083c4da6-8c2b-442a-a286-93d5ac659bca","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:29:12.379Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"083c4da6-8c2b-442a-a286-93d5ac659bca","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:29:12.379Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"083c4da6-8c2b-442a-a286-93d5ac659bca","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:29:12.379Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"083c4da6-8c2b-442a-a286-93d5ac659bca","podName":"delete-head-after-submit-gg2wh-small-group-worker-dxpfb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:29:12.379Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"083c4da6-8c2b-442a-a286-93d5ac659bca","podName":"delete-head-after-submit-gg2wh-small-group-worker-dxpfb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:29:12.379Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"083c4da6-8c2b-442a-a286-93d5ac659bca","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:29:12.379Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"083c4da6-8c2b-442a-a286-93d5ac659bca","seconds":300}
{"level":"info","ts":"2026-11-03T15:29:13.640Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"36f5e075-fe3f-428b-8b91-a2fcedfb85b4","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:29:13.640Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"36f5e075-fe3f-428b-8b91-a2fcedfb85b4","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:29:13.640Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"36f5e075-fe3f-428b-8b91-a2fcedfb85b4","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:29:13.640Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"36f5e075-fe3f-428b-8b91-a2fcedfb85b4"}
{"level":"info","ts":"2026-11-03T15:29:13.640Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"36f5e075-fe3f-428b-8b91-a2fcedfb85b4","RayCluster":"delete-head-after-submit-gg2wh","State":""}
{"level":"info","ts":"2026-11-03T15:29:13.640Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"36f5e075-fe3f-428b-8b91-a2fcedfb85b4","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:29:01Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:29:01Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:29:01Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:29:01Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:29:15.543Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"0eb0f364-1637-4b21-94a5-3dcea5d25ffa"}
{"level":"info","ts":"2026-11-03T15:29:15.543Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"0eb0f364-1637-4b21-94a5-3dcea5d25ffa"}
{"level":"info","ts":"2026-11-03T15:29:15.543Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"0eb0f364-1637-4b21-94a5-3dcea5d25ffa","1 head service found":"long-running-hvml9-head-svc"}
{"level":"info","ts":"2026-11-03T15:29:15.544Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"0eb0f364-1637-4b21-94a5-3dcea5d25ffa","Found 1 head Pod":"long-running-hvml9-head-s8j5v","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:29:15.544Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"0eb0f364-1637-4b21-94a5-3dcea5d25ffa","head Pod":"long-running-hvml9-head-s8j5v","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-hvml9-head-s8j5v. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:29:15.544Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"0eb0f364-1637-4b21-94a5-3dcea5d25ffa","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:29:15.544Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"0eb0f364-1637-4b21-94a5-3dcea5d25ffa","worker Pod":"long-running-hvml9-small-group-worker-mjl4n","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-hvml9-small-group-worker-mjl4n. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:29:15.544Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"0eb0f364-1637-4b21-94a5-3dcea5d25ffa","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:29:15.544Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"0eb0f364-1637-4b21-94a5-3dcea5d25ffa","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:29:15.544Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"0eb0f364-1637-4b21-94a5-3dcea5d25ffa","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:29:15.544Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"0eb0f364-1637-4b21-94a5-3dcea5d25ffa","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:29:15.544Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"0eb0f364-1637-4b21-94a5-3dcea5d25ffa","podName":"long-running-hvml9-small-group-worker-mjl4n","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:29:15.544Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"0eb0f364-1637-4b21-94a5-3dcea5d25ffa","podName":"long-running-hvml9-small-group-worker-mjl4n","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:29:15.544Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"0eb0f364-1637-4b21-94a5-3dcea5d25ffa","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:29:15.544Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"0eb0f364-1637-4b21-94a5-3dcea5d25ffa","seconds":300}
{"level":"info","ts":"2026-11-03T15:29:16.564Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"e30a76f9-27d0-44d2-8a34-8cd0880c8669"}
{"level":"info","ts":"2026-11-03T15:29:16.564Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"e30a76f9-27d0-44d2-8a34-8cd0880c8669"}
{"level":"info","ts":"2026-11-03T15:29:16.564Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"e30a76f9-27d0-44d2-8a34-8cd0880c8669","1 head service found":"long-running-hvml9-head-svc"}
{"level":"info","ts":"2026-11-03T15:29:16.564Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"e30a76f9-27d0-44d2-8a34-8cd0880c8669","Found 1 head Pod":"long-running-hvml9-head-s8j5v","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:29:16.564Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"e30a76f9-27d0-44d2-8a34-8cd0880c8669","head Pod":"long-running-hvml9-head-s8j5v","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-hvml9-head-s8j5v. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:29:16.564Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"e30a76f9-27d0-44d2-8a34-8cd0880c8669","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:29:16.564Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"e30a76f9-27d0-44d2-8a34-8cd0880c8669","worker Pod":"long-running-hvml9-small-group-worker-mjl4n","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-hvml9-small-group-worker-mjl4n. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:29:16.564Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"e30a76f9-27d0-44d2-8a34-8cd0880c8669","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:29:16.564Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"e30a76f9-27d0-44d2-8a34-8cd0880c8669","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:29:16.564Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"e30a76f9-27d0-44d2-8a34-8cd0880c8669","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:29:16.564Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"e30a76f9-27d0-44d2-8a34-8cd0880c8669","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:29:16.564Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"e30a76f9-27d0-44d2-8a34-8cd0880c8669","podName":"long-running-hvml9-small-group-worker-mjl4n","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:29:16.564Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"e30a76f9-27d0-44d2-8a34-8cd0880c8669","podName":"long-running-hvml9-small-group-worker-mjl4n","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:29:16.564Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"e30a76f9-27d0-44d2-8a34-8cd0880c8669","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:29:16.564Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"e30a76f9-27d0-44d2-8a34-8cd0880c8669","seconds":300}
{"level":"info","ts":"2026-11-03T15:29:16.640Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"e62f0c7b-f759-446c-9ba7-6aa0e0e4798e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:29:16.640Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"e62f0c7b-f759-446c-9ba7-6aa0e0e4798e","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:29:16.640Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"e62f0c7b-f759-446c-9ba7-6aa0e0e4798e","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:29:16.641Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"e62f0c7b-f759-446c-9ba7-6aa0e0e4798e"}
{"level":"info","ts":"2026-11-03T15:29:16.641Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"e62f0c7b-f759-446c-9ba7-6aa0e0e4798e","RayCluster":"delete-head-after-submit-gg2wh","State":""}
{"level":"info","ts":"2026-11-03T15:29:16.641Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"e62f0c7b-f759-446c-9ba7-6aa0e0e4798e","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:29:01Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:29:01Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:29:01Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:29:01Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:29:19.642Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"27dde0b0-be0a-4c48-8a5d-64f07f6370f9","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:29:19.642Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"27dde0b0-be0a-4c48-8a5d-64f07f6370f9","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:29:19.642Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"27dde0b0-be0a-4c48-8a5d-64f07f6370f9","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:29:19.642Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"27dde0b0-be0a-4c48-8a5d-64f07f6370f9"}
{"level":"info","ts":"2026-11-03T15:29:19.642Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"27dde0b0-be0a-4c48-8a5d-64f07f6370f9","RayCluster":"delete-head-after-submit-gg2wh","State":""}
{"level":"info","ts":"2026-11-03T15:29:19.642Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"27dde0b0-be0a-4c48-8a5d-64f07f6370f9","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:29:01Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:29:01Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:29:01Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:29:01Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:29:21.417Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"f74a1415-11dd-4794-a5c6-b14b21ce320b"}
{"level":"info","ts":"2026-11-03T15:29:21.417Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"f74a1415-11dd-4794-a5c6-b14b21ce320b"}
{"level":"info","ts":"2026-11-03T15:29:21.417Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"f74a1415-11dd-4794-a5c6-b14b21ce320b","1 head service found":"delete-head-after-submit-gg2wh-head-svc"}
{"level":"info","ts":"2026-11-03T15:29:21.417Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"f74a1415-11dd-4794-a5c6-b14b21ce320b","Found 1 head Pod":"delete-head-after-submit-gg2wh-head-b99bx","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:29:21.417Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"f74a1415-11dd-4794-a5c6-b14b21ce320b","head Pod":"delete-head-after-submit-gg2wh-head-b99bx","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod delete-head-after-submit-gg2wh-head-b99bx. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:29:21.417Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"f74a1415-11dd-4794-a5c6-b14b21ce320b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:29:21.417Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"f74a1415-11dd-4794-a5c6-b14b21ce320b","worker Pod":"delete-head-after-submit-gg2wh-small-group-worker-dxpfb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod delete-head-after-submit-gg2wh-small-group-worker-dxpfb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:29:21.417Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"f74a1415-11dd-4794-a5c6-b14b21ce320b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:29:21.417Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"f74a1415-11dd-4794-a5c6-b14b21ce320b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:29:21.417Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"f74a1415-11dd-4794-a5c6-b14b21ce320b","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:29:21.417Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"f74a1415-11dd-4794-a5c6-b14b21ce320b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:29:21.417Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"f74a1415-11dd-4794-a5c6-b14b21ce320b","podName":"delete-head-after-submit-gg2wh-small-group-worker-dxpfb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:29:21.417Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"f74a1415-11dd-4794-a5c6-b14b21ce320b","podName":"delete-head-after-submit-gg2wh-small-group-worker-dxpfb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:29:21.418Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"f74a1415-11dd-4794-a5c6-b14b21ce320b","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:29:21.418Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"f74a1415-11dd-4794-a5c6-b14b21ce320b","seconds":300}
{"level":"info","ts":"2026-11-03T15:29:22.472Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"8f7ea78e-949b-4a15-b4cb-02fcee1629b0"}
{"level":"info","ts":"2026-11-03T15:29:22.472Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"8f7ea78e-949b-4a15-b4cb-02fcee1629b0"}
{"level":"info","ts":"2026-11-03T15:29:22.472Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"8f7ea78e-949b-4a15-b4cb-02fcee1629b0","1 head service found":"delete-head-after-submit-gg2wh-head-svc"}
{"level":"info","ts":"2026-11-03T15:29:22.472Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"8f7ea78e-949b-4a15-b4cb-02fcee1629b0","Found 1 head Pod":"delete-head-after-submit-gg2wh-head-b99bx","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:29:22.472Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"8f7ea78e-949b-4a15-b4cb-02fcee1629b0","head Pod":"delete-head-after-submit-gg2wh-head-b99bx","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod delete-head-after-submit-gg2wh-head-b99bx. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:29:22.472Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"8f7ea78e-949b-4a15-b4cb-02fcee1629b0","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:29:22.472Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"8f7ea78e-949b-4a15-b4cb-02fcee1629b0","worker Pod":"delete-head-after-submit-gg2wh-small-group-worker-dxpfb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod delete-head-after-submit-gg2wh-small-group-worker-dxpfb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:29:22.472Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"8f7ea78e-949b-4a15-b4cb-02fcee1629b0","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:29:22.472Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"8f7ea78e-949b-4a15-b4cb-02fcee1629b0","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:29:22.472Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"8f7ea78e-949b-4a15-b4cb-02fcee1629b0","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:29:22.472Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"8f7ea78e-949b-4a15-b4cb-02fcee1629b0","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:29:22.473Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"8f7ea78e-949b-4a15-b4cb-02fcee1629b0","podName":"delete-head-after-submit-gg2wh-small-group-worker-dxpfb","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:27:43Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:29:22.473Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"8f7ea78e-949b-4a15-b4cb-02fcee1629b0","podName":"delete-head-after-submit-gg2wh-small-group-worker-dxpfb","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:27:43Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:29:22.473Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"8f7ea78e-949b-4a15-b4cb-02fcee1629b0","name":"delete-head-after-submit-gg2wh","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:29:01Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:29:01Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:29:22Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:29:01Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:29:22.534Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"0b295026-6cab-4b02-8b1e-4d8ab231302f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:29:22.534Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"0b295026-6cab-4b02-8b1e-4d8ab231302f","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:29:22.534Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"ece066e4-e83a-486e-90ea-8215d502e1ce","cluster":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:29:22.535Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"0b295026-6cab-4b02-8b1e-4d8ab231302f","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:29:22.535Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"0b295026-6cab-4b02-8b1e-4d8ab231302f"}
{"level":"info","ts":"2026-11-03T15:29:22.535Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"0b295026-6cab-4b02-8b1e-4d8ab231302f","RayCluster":"delete-head-after-submit-gg2wh","State":""}
{"level":"info","ts":"2026-11-03T15:29:22.535Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"0b295026-6cab-4b02-8b1e-4d8ab231302f","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:29:01Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:29:01Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:29:22Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:29:01Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:29:22.535Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"0b295026-6cab-4b02-8b1e-4d8ab231302f","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:29:22.596Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"bc95b389-3743-486f-bd7e-852dfa697ab3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:29:22.596Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"bc95b389-3743-486f-bd7e-852dfa697ab3","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:29:22.596Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"bc95b389-3743-486f-bd7e-852dfa697ab3","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:29:22.596Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"bc95b389-3743-486f-bd7e-852dfa697ab3"}
{"level":"info","ts":"2026-11-03T15:29:22.596Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"bc95b389-3743-486f-bd7e-852dfa697ab3","RayCluster":"delete-head-after-submit-gg2wh","State":""}
{"level":"info","ts":"2026-11-03T15:29:22.596Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"bc95b389-3743-486f-bd7e-852dfa697ab3","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:29:22Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:29:01Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:29:22Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:29:01Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:29:24.535Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"b5c55e27-e761-4e3c-afe9-6e602bea6853"}
{"level":"info","ts":"2026-11-03T15:29:24.535Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"b5c55e27-e761-4e3c-afe9-6e602bea6853"}
{"level":"info","ts":"2026-11-03T15:29:24.535Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"b5c55e27-e761-4e3c-afe9-6e602bea6853","1 head service found":"delete-head-after-submit-gg2wh-head-svc"}
{"level":"info","ts":"2026-11-03T15:29:24.535Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"b5c55e27-e761-4e3c-afe9-6e602bea6853","Found 1 head Pod":"delete-head-after-submit-gg2wh-head-b99bx","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:29:24.535Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"b5c55e27-e761-4e3c-afe9-6e602bea6853","head Pod":"delete-head-after-submit-gg2wh-head-b99bx","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod delete-head-after-submit-gg2wh-head-b99bx. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:29:24.535Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"b5c55e27-e761-4e3c-afe9-6e602bea6853","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:29:24.535Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"b5c55e27-e761-4e3c-afe9-6e602bea6853","worker Pod":"delete-head-after-submit-gg2wh-small-group-worker-dxpfb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod delete-head-after-submit-gg2wh-small-group-worker-dxpfb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:29:24.535Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"b5c55e27-e761-4e3c-afe9-6e602bea6853","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:29:24.535Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"b5c55e27-e761-4e3c-afe9-6e602bea6853","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:29:24.535Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"b5c55e27-e761-4e3c-afe9-6e602bea6853","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:29:24.535Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"b5c55e27-e761-4e3c-afe9-6e602bea6853","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:29:24.535Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"b5c55e27-e761-4e3c-afe9-6e602bea6853","podName":"delete-head-after-submit-gg2wh-small-group-worker-dxpfb","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:27:43Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:29:24.536Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"b5c55e27-e761-4e3c-afe9-6e602bea6853","podName":"delete-head-after-submit-gg2wh-small-group-worker-dxpfb","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:27:43Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:29:24.536Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"b5c55e27-e761-4e3c-afe9-6e602bea6853","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:29:24.536Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"b5c55e27-e761-4e3c-afe9-6e602bea6853","seconds":300}
{"level":"info","ts":"2026-11-03T15:29:25.597Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"8ae0ceb9-f74e-4d7e-84af-21ea125ba52e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:29:25.597Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"8ae0ceb9-f74e-4d7e-84af-21ea125ba52e","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:29:25.597Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"8ae0ceb9-f74e-4d7e-84af-21ea125ba52e","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:29:25.597Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"8ae0ceb9-f74e-4d7e-84af-21ea125ba52e"}
{"level":"info","ts":"2026-11-03T15:29:25.597Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"8ae0ceb9-f74e-4d7e-84af-21ea125ba52e","RayCluster":"delete-head-after-submit-gg2wh","State":""}
{"level":"info","ts":"2026-11-03T15:29:25.597Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"8ae0ceb9-f74e-4d7e-84af-21ea125ba52e","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:29:22Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:29:01Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:29:22Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:29:01Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:29:25.604Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"b522d364-97be-4e18-928c-71943210547d"}
{"level":"info","ts":"2026-11-03T15:29:25.604Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"b522d364-97be-4e18-928c-71943210547d"}
{"level":"info","ts":"2026-11-03T15:29:25.604Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"b522d364-97be-4e18-928c-71943210547d","1 head service found":"long-running-hvml9-head-svc"}
{"level":"info","ts":"2026-11-03T15:29:25.604Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"b522d364-97be-4e18-928c-71943210547d","Found 1 head Pod":"long-running-hvml9-head-s8j5v","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:29:25.604Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"b522d364-97be-4e18-928c-71943210547d","head Pod":"long-running-hvml9-head-s8j5v","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-hvml9-head-s8j5v. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:29:25.604Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"b522d364-97be-4e18-928c-71943210547d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:29:25.604Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"b522d364-97be-4e18-928c-71943210547d","worker Pod":"long-running-hvml9-small-group-worker-mjl4n","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-hvml9-small-group-worker-mjl4n. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:29:25.604Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"b522d364-97be-4e18-928c-71943210547d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:29:25.604Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"b522d364-97be-4e18-928c-71943210547d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:29:25.604Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"b522d364-97be-4e18-928c-71943210547d","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:29:25.604Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"b522d364-97be-4e18-928c-71943210547d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:29:25.604Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"b522d364-97be-4e18-928c-71943210547d","podName":"long-running-hvml9-small-group-worker-mjl4n","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:29:25.604Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"b522d364-97be-4e18-928c-71943210547d","podName":"long-running-hvml9-small-group-worker-mjl4n","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:29:25.604Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"b522d364-97be-4e18-928c-71943210547d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:29:25.604Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"b522d364-97be-4e18-928c-71943210547d","seconds":300}
{"level":"info","ts":"2026-11-03T15:29:26.658Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"259b8840-a451-4a0c-b718-5abb00e64da9"}
{"level":"info","ts":"2026-11-03T15:29:26.658Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"259b8840-a451-4a0c-b718-5abb00e64da9"}
{"level":"info","ts":"2026-11-03T15:29:26.658Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"259b8840-a451-4a0c-b718-5abb00e64da9","1 head service found":"long-running-hvml9-head-svc"}
{"level":"info","ts":"2026-11-03T15:29:26.658Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"259b8840-a451-4a0c-b718-5abb00e64da9","Found 1 head Pod":"long-running-hvml9-head-s8j5v","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:29:26.658Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"259b8840-a451-4a0c-b718-5abb00e64da9","head Pod":"long-running-hvml9-head-s8j5v","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-hvml9-head-s8j5v. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:29:26.658Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"259b8840-a451-4a0c-b718-5abb00e64da9","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:29:26.658Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"259b8840-a451-4a0c-b718-5abb00e64da9","worker Pod":"long-running-hvml9-small-group-worker-mjl4n","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-hvml9-small-group-worker-mjl4n. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:29:26.658Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"259b8840-a451-4a0c-b718-5abb00e64da9","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:29:26.658Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"259b8840-a451-4a0c-b718-5abb00e64da9","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:29:26.658Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"259b8840-a451-4a0c-b718-5abb00e64da9","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:29:26.658Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"259b8840-a451-4a0c-b718-5abb00e64da9","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:29:26.658Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"259b8840-a451-4a0c-b718-5abb00e64da9","podName":"long-running-hvml9-small-group-worker-mjl4n","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:27:42Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:29:26.659Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"259b8840-a451-4a0c-b718-5abb00e64da9","podName":"long-running-hvml9-small-group-worker-mjl4n","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:27:42Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:29:26.659Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"259b8840-a451-4a0c-b718-5abb00e64da9","name":"long-running-hvml9","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:29:06Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.42","serviceIP":"10.132.0.42","podName":"long-running-hvml9-head-s8j5v","serviceName":"long-running-hvml9-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:29:06Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:32Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:29:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.42","serviceIP":"10.132.0.42","podName":"long-running-hvml9-head-s8j5v","serviceName":"long-running-hvml9-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:29:06Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:32Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:29:26.720Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"bab0664b-91ff-47fb-afed-4b539bfa0463","cluster":"long-running-hvml9"}
{"level":"info","ts":"2026-11-03T15:29:26.720Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"95e4d28e-6842-46cc-b44d-2108779447c9","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:29:26.720Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"95e4d28e-6842-46cc-b44d-2108779447c9","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Failed","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":600,"Status.endTime":"2026-11-03 15:27:38 +0000 UTC","Now":"2026-11-03T15:29:26.720Z","ShutdownTime":"2026-11-03T15:37:38.000Z"}
{"level":"info","ts":"2026-11-03T15:29:26.720Z","logger":"controllers.RayJob","msg":"TTL has not been met for ShutdownAfterJobFinishes. Requeuing.","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"95e4d28e-6842-46cc-b44d-2108779447c9","deletionMechanism":"ShutdownAfterJobFinishes","shutdownTime":"2026-11-03T15:37:38.000Z","requeueAfter":493.279715207}
{"level":"info","ts":"2026-11-03T15:29:28.598Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"fd4b6ccd-7054-47e5-a8ef-7cebec46af96","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:29:28.598Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"fd4b6ccd-7054-47e5-a8ef-7cebec46af96","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:29:28.598Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"fd4b6ccd-7054-47e5-a8ef-7cebec46af96","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:29:28.599Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"fd4b6ccd-7054-47e5-a8ef-7cebec46af96"}
{"level":"info","ts":"2026-11-03T15:29:28.599Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"fd4b6ccd-7054-47e5-a8ef-7cebec46af96","RayCluster":"delete-head-after-submit-gg2wh","State":""}
{"level":"info","ts":"2026-11-03T15:29:28.599Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"fd4b6ccd-7054-47e5-a8ef-7cebec46af96","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:29:22Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:29:01Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:29:22Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:29:01Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:29:28.720Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"6acad64c-98b4-404a-ae1a-d262c87b8a82"}
{"level":"info","ts":"2026-11-03T15:29:28.720Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"6acad64c-98b4-404a-ae1a-d262c87b8a82"}
{"level":"info","ts":"2026-11-03T15:29:28.720Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"6acad64c-98b4-404a-ae1a-d262c87b8a82","1 head service found":"long-running-hvml9-head-svc"}
{"level":"info","ts":"2026-11-03T15:29:28.720Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"6acad64c-98b4-404a-ae1a-d262c87b8a82","Found 1 head Pod":"long-running-hvml9-head-s8j5v","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:29:28.720Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"6acad64c-98b4-404a-ae1a-d262c87b8a82","head Pod":"long-running-hvml9-head-s8j5v","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-hvml9-head-s8j5v. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:29:28.720Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"6acad64c-98b4-404a-ae1a-d262c87b8a82","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:29:28.720Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"6acad64c-98b4-404a-ae1a-d262c87b8a82","worker Pod":"long-running-hvml9-small-group-worker-mjl4n","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-hvml9-small-group-worker-mjl4n. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:29:28.720Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"6acad64c-98b4-404a-ae1a-d262c87b8a82","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:29:28.720Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"6acad64c-98b4-404a-ae1a-d262c87b8a82","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:29:28.720Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"6acad64c-98b4-404a-ae1a-d262c87b8a82","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:29:28.720Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"6acad64c-98b4-404a-ae1a-d262c87b8a82","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:29:28.720Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"6acad64c-98b4-404a-ae1a-d262c87b8a82","podName":"long-running-hvml9-small-group-worker-mjl4n","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:27:42Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:29:28.720Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"6acad64c-98b4-404a-ae1a-d262c87b8a82","podName":"long-running-hvml9-small-group-worker-mjl4n","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:27:42Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:29:28.720Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"6acad64c-98b4-404a-ae1a-d262c87b8a82","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:29:28.720Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"6acad64c-98b4-404a-ae1a-d262c87b8a82","seconds":300}
{"level":"info","ts":"2026-11-03T15:29:31.600Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"7d6aaa82-db27-496b-be96-a167606b5130","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:29:31.600Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"7d6aaa82-db27-496b-be96-a167606b5130","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:29:31.600Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"7d6aaa82-db27-496b-be96-a167606b5130","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:29:31.600Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"7d6aaa82-db27-496b-be96-a167606b5130"}
{"level":"info","ts":"2026-11-03T15:29:31.600Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"7d6aaa82-db27-496b-be96-a167606b5130","RayCluster":"delete-head-after-submit-gg2wh","State":""}
{"level":"info","ts":"2026-11-03T15:29:31.600Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"7d6aaa82-db27-496b-be96-a167606b5130","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:29:22Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:29:01Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:29:22Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:29:01Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:29:33.481Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"a7d2c011-11d3-443b-9313-cff83898d745"}
{"level":"info","ts":"2026-11-03T15:29:33.481Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"a7d2c011-11d3-443b-9313-cff83898d745"}
{"level":"info","ts":"2026-11-03T15:29:33.481Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"a7d2c011-11d3-443b-9313-cff83898d745","1 head service found":"delete-head-after-submit-gg2wh-head-svc"}
{"level":"info","ts":"2026-11-03T15:29:33.481Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"a7d2c011-11d3-443b-9313-cff83898d745","Found 1 head Pod":"delete-head-after-submit-gg2wh-head-b99bx","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:29:33.482Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"a7d2c011-11d3-443b-9313-cff83898d745","head Pod":"delete-head-after-submit-gg2wh-head-b99bx","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod delete-head-after-submit-gg2wh-head-b99bx. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:29:33.482Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"a7d2c011-11d3-443b-9313-cff83898d745","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:29:33.482Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"a7d2c011-11d3-443b-9313-cff83898d745","worker Pod":"delete-head-after-submit-gg2wh-small-group-worker-dxpfb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod delete-head-after-submit-gg2wh-small-group-worker-dxpfb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:29:33.482Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"a7d2c011-11d3-443b-9313-cff83898d745","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:29:33.482Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"a7d2c011-11d3-443b-9313-cff83898d745","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:29:33.482Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"a7d2c011-11d3-443b-9313-cff83898d745","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:29:33.482Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"a7d2c011-11d3-443b-9313-cff83898d745","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:29:33.482Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"a7d2c011-11d3-443b-9313-cff83898d745","name":"delete-head-after-submit-gg2wh","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:29:22Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:29:01Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:29:33Z","stateTransitionTimes":{"ready":"2026-11-03T15:29:33Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:29:01Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:29:33Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:29:33.543Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"c08ad05e-45a8-4d21-a44a-3b36f97e98f7","cluster":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:29:33.543Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"313b1101-41ff-46df-bf27-ba0fa768311e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:29:33.543Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"313b1101-41ff-46df-bf27-ba0fa768311e","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:29:33.543Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"313b1101-41ff-46df-bf27-ba0fa768311e","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:29:33.543Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"313b1101-41ff-46df-bf27-ba0fa768311e"}
{"level":"info","ts":"2026-11-03T15:29:33.607Z","logger":"controllers.RayJob","msg":"Created submitter Kubernetes Job for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"313b1101-41ff-46df-bf27-ba0fa768311e","Kubernetes Job":"delete-head-after-submit"}
{"level":"info","ts":"2026-11-03T15:29:33.607Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"313b1101-41ff-46df-bf27-ba0fa768311e","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:29:22Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:29:01Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","dashboardURL":"delete-head-after-submit-gg2wh-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:29:22Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:29:01Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:29:33.607Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"313b1101-41ff-46df-bf27-ba0fa768311e","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-11-03T15:29:33.667Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"8b097e77-ff95-46cf-877e-5052e9484436","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:29:33.667Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"8b097e77-ff95-46cf-877e-5052e9484436","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:29:33.667Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"8b097e77-ff95-46cf-877e-5052e9484436","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:29:33.667Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"8b097e77-ff95-46cf-877e-5052e9484436"}
{"level":"info","ts":"2026-11-03T15:29:33.668Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job for RayJob already exists","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"8b097e77-ff95-46cf-877e-5052e9484436","Kubernetes Job":"delete-head-after-submit"}
{"level":"info","ts":"2026-11-03T15:29:33.668Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"8b097e77-ff95-46cf-877e-5052e9484436","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:29:22Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:29:01Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","dashboardURL":"delete-head-after-submit-gg2wh-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:29:22Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:29:01Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:29:33.668Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"8b097e77-ff95-46cf-877e-5052e9484436","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-11-03T15:29:33.722Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"8b097e77-ff95-46cf-877e-5052e9484436","error":"Operation cannot be fulfilled on rayjobs.ray.io \"delete-head-after-submit\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-11-03T15:29:33.722Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"8b097e77-ff95-46cf-877e-5052e9484436"}
{"level":"error","ts":"2026-11-03T15:29:33.722Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"8b097e77-ff95-46cf-877e-5052e9484436","error":"Operation cannot be fulfilled on rayjobs.ray.io \"delete-head-after-submit\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:29:33.722Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"97df8fee-c6f0-45af-bab7-cd2f79d99ba8","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:29:33.722Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"97df8fee-c6f0-45af-bab7-cd2f79d99ba8","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:29:33.722Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"97df8fee-c6f0-45af-bab7-cd2f79d99ba8","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:29:33.723Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"97df8fee-c6f0-45af-bab7-cd2f79d99ba8"}
{"level":"info","ts":"2026-11-03T15:29:33.723Z","logger":"controllers.RayJob","msg":"Create a Kubernetes Service","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"97df8fee-c6f0-45af-bab7-cd2f79d99ba8","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:29:33.786Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"0aaf232a-3e62-4b5a-abd9-9c3872c8e536","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:29:33.786Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"0aaf232a-3e62-4b5a-abd9-9c3872c8e536","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:29:33.786Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"0aaf232a-3e62-4b5a-abd9-9c3872c8e536","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:29:33.786Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"0aaf232a-3e62-4b5a-abd9-9c3872c8e536"}
{"level":"info","ts":"2026-11-03T15:29:33.787Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"0aaf232a-3e62-4b5a-abd9-9c3872c8e536","rayCluster":"delete-head-after-submit-gg2wh","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:29:34.742Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"ce813cc5-54d5-4718-bc73-65a2d893159d"}
{"level":"info","ts":"2026-11-03T15:29:34.742Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"ce813cc5-54d5-4718-bc73-65a2d893159d"}
{"level":"info","ts":"2026-11-03T15:29:34.742Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"ce813cc5-54d5-4718-bc73-65a2d893159d","1 head service found":"delete-head-after-submit-gg2wh-head-svc"}
{"level":"info","ts":"2026-11-03T15:29:34.742Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"ce813cc5-54d5-4718-bc73-65a2d893159d","Found 1 head Pod":"delete-head-after-submit-gg2wh-head-b99bx","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:29:34.742Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"ce813cc5-54d5-4718-bc73-65a2d893159d","head Pod":"delete-head-after-submit-gg2wh-head-b99bx","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod delete-head-after-submit-gg2wh-head-b99bx. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:29:34.742Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"ce813cc5-54d5-4718-bc73-65a2d893159d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:29:34.742Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"ce813cc5-54d5-4718-bc73-65a2d893159d","worker Pod":"delete-head-after-submit-gg2wh-small-group-worker-dxpfb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod delete-head-after-submit-gg2wh-small-group-worker-dxpfb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:29:34.742Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"ce813cc5-54d5-4718-bc73-65a2d893159d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:29:34.742Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"ce813cc5-54d5-4718-bc73-65a2d893159d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:29:34.742Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"ce813cc5-54d5-4718-bc73-65a2d893159d","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:29:34.742Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"ce813cc5-54d5-4718-bc73-65a2d893159d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:29:34.743Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"ce813cc5-54d5-4718-bc73-65a2d893159d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:29:34.743Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"ce813cc5-54d5-4718-bc73-65a2d893159d","seconds":300}
{"level":"info","ts":"2026-11-03T15:29:34.903Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"4c43c3ba-b9bd-4e8f-bf6e-d74bdaac4782"}
{"level":"info","ts":"2026-11-03T15:29:34.903Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"4c43c3ba-b9bd-4e8f-bf6e-d74bdaac4782"}
{"level":"info","ts":"2026-11-03T15:29:34.903Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"4c43c3ba-b9bd-4e8f-bf6e-d74bdaac4782","1 head service found":"delete-head-after-submit-gg2wh-head-svc"}
{"level":"info","ts":"2026-11-03T15:29:34.903Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"4c43c3ba-b9bd-4e8f-bf6e-d74bdaac4782","Found 1 head Pod":"delete-head-after-submit-gg2wh-head-b99bx","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:29:34.904Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"4c43c3ba-b9bd-4e8f-bf6e-d74bdaac4782","head Pod":"delete-head-after-submit-gg2wh-head-b99bx","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod delete-head-after-submit-gg2wh-head-b99bx. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:29:34.904Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"4c43c3ba-b9bd-4e8f-bf6e-d74bdaac4782","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:29:34.904Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"4c43c3ba-b9bd-4e8f-bf6e-d74bdaac4782","worker Pod":"delete-head-after-submit-gg2wh-small-group-worker-dxpfb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod delete-head-after-submit-gg2wh-small-group-worker-dxpfb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:29:34.904Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"4c43c3ba-b9bd-4e8f-bf6e-d74bdaac4782","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:29:34.904Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"4c43c3ba-b9bd-4e8f-bf6e-d74bdaac4782","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:29:34.904Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"4c43c3ba-b9bd-4e8f-bf6e-d74bdaac4782","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:29:34.904Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"4c43c3ba-b9bd-4e8f-bf6e-d74bdaac4782","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:29:34.904Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"4c43c3ba-b9bd-4e8f-bf6e-d74bdaac4782","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:29:34.904Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"4c43c3ba-b9bd-4e8f-bf6e-d74bdaac4782","seconds":300}
{"level":"info","ts":"2026-11-03T15:29:36.436Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"c60a42a4-08d5-4d64-81bc-fce82c1550ad","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:29:36.436Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"c60a42a4-08d5-4d64-81bc-fce82c1550ad","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:29:36.436Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"c60a42a4-08d5-4d64-81bc-fce82c1550ad","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:29:36.436Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"c60a42a4-08d5-4d64-81bc-fce82c1550ad"}
{"level":"info","ts":"2026-11-03T15:29:36.436Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"c60a42a4-08d5-4d64-81bc-fce82c1550ad","rayCluster":"delete-head-after-submit-gg2wh","serviceType":"headService"}
{"level":"error","ts":"2026-11-03T15:29:36.438Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"c60a42a4-08d5-4d64-81bc-fce82c1550ad","JobId":"delete-head-after-submit-cwfp8","error":"Get \"http://delete-head-after-submit-gg2wh-head-svc.test-ns-kbx4w.svc.cluster.local:8265/api/jobs/delete-head-after-submit-cwfp8\": dial tcp 10.134.0.48:8265: connect: connection refused","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:321\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-03T15:29:36.438Z","logger":"controllers.RayJob","msg":"Job status check began failing.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"c60a42a4-08d5-4d64-81bc-fce82c1550ad","JobId":"delete-head-after-submit-cwfp8","error":"Get \"http://delete-head-after-submit-gg2wh-head-svc.test-ns-kbx4w.svc.cluster.local:8265/api/jobs/delete-head-after-submit-cwfp8\": dial tcp 10.134.0.48:8265: connect: connection refused","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.recordJobStatusCheckFailure\n\t/workspace/controllers/ray/rayjob_controller.go:1381\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:322\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:29:36.439Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"c60a42a4-08d5-4d64-81bc-fce82c1550ad","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","dashboardURL":"delete-head-after-submit-gg2wh-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:29:22Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:29:01Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","dashboardURL":"delete-head-after-submit-gg2wh-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:29:22Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:29:01Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"jobStatusCheckFailureStartTime":"2026-11-03T15:29:36Z"}}
{"level":"info","ts":"2026-11-03T15:29:36.439Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"c60a42a4-08d5-4d64-81bc-fce82c1550ad","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-11-03T15:29:36.536Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"199ee445-bbfd-4130-9947-7bfe4edf7689","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:29:36.536Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"199ee445-bbfd-4130-9947-7bfe4edf7689","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:29:36.536Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"199ee445-bbfd-4130-9947-7bfe4edf7689","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:29:36.537Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"199ee445-bbfd-4130-9947-7bfe4edf7689"}
{"level":"info","ts":"2026-11-03T15:29:36.537Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"199ee445-bbfd-4130-9947-7bfe4edf7689","rayCluster":"delete-head-after-submit-gg2wh","serviceType":"headService"}
{"level":"error","ts":"2026-11-03T15:29:36.539Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"199ee445-bbfd-4130-9947-7bfe4edf7689","JobId":"delete-head-after-submit-cwfp8","error":"Get \"http://delete-head-after-submit-gg2wh-head-svc.test-ns-kbx4w.svc.cluster.local:8265/api/jobs/delete-head-after-submit-cwfp8\": dial tcp 10.134.0.48:8265: connect: connection refused","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:321\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:29:37.191Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"7fe59ad6-12d8-49e3-8eeb-ed21937fa452"}
{"level":"info","ts":"2026-11-03T15:29:37.191Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"7fe59ad6-12d8-49e3-8eeb-ed21937fa452"}
{"level":"info","ts":"2026-11-03T15:29:37.191Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"7fe59ad6-12d8-49e3-8eeb-ed21937fa452","1 head service found":"delete-head-after-submit-gg2wh-head-svc"}
{"level":"info","ts":"2026-11-03T15:29:37.191Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"7fe59ad6-12d8-49e3-8eeb-ed21937fa452","Found 1 head Pod":"delete-head-after-submit-gg2wh-head-b99bx","Pod status":"Failed","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"&ContainerStateTerminated{ExitCode:1,Signal:0,Reason:Error,Message:,StartedAt:2026-11-03 15:28:49 +0000 UTC,FinishedAt:2026-11-03 15:29:36 +0000 UTC,ContainerID:cri-o://68c08884e0855ffb9c397151e03677f577d8f32d76b36ea0c9ee06ca5dbdf315,}"}
{"level":"info","ts":"2026-11-03T15:29:37.191Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"7fe59ad6-12d8-49e3-8eeb-ed21937fa452","head Pod":"delete-head-after-submit-gg2wh-head-b99bx","shouldDelete":true,"reason":"The head Pod delete-head-after-submit-gg2wh-head-b99bx status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary."}
{"level":"error","ts":"2026-11-03T15:29:37.241Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"7fe59ad6-12d8-49e3-8eeb-ed21937fa452","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"The head Pod delete-head-after-submit-gg2wh-head-b99bx status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary.","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:29:37.241Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"7fe59ad6-12d8-49e3-8eeb-ed21937fa452","error":"The head Pod delete-head-after-submit-gg2wh-head-b99bx status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary."}
{"level":"info","ts":"2026-11-03T15:29:37.241Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"7fe59ad6-12d8-49e3-8eeb-ed21937fa452","name":"delete-head-after-submit-gg2wh","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:29:33Z","stateTransitionTimes":{"ready":"2026-11-03T15:29:33Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:29:01Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:29:33Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:29:37Z","stateTransitionTimes":{"ready":"2026-11-03T15:29:33Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:29:37Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:29:33Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:29:37.302Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"7fe59ad6-12d8-49e3-8eeb-ed21937fa452"}
{"level":"error","ts":"2026-11-03T15:29:37.302Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"7fe59ad6-12d8-49e3-8eeb-ed21937fa452","error":"The head Pod delete-head-after-submit-gg2wh-head-b99bx status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary.","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:29:37.303Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"342c0bf1-a19f-44cd-8a07-79807bfaf77a","cluster":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:29:37.303Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"af7f7255-ed01-4e71-b945-0e1b2e5b55f2","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:29:37.303Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"af7f7255-ed01-4e71-b945-0e1b2e5b55f2","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:29:37.303Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"af7f7255-ed01-4e71-b945-0e1b2e5b55f2","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:29:37.303Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"af7f7255-ed01-4e71-b945-0e1b2e5b55f2"}
{"level":"info","ts":"2026-11-03T15:29:37.303Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"af7f7255-ed01-4e71-b945-0e1b2e5b55f2","rayCluster":"delete-head-after-submit-gg2wh","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:29:37.309Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"5925b5bf-1473-4c8e-a859-a3f141974026"}
{"level":"info","ts":"2026-11-03T15:29:37.309Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"5925b5bf-1473-4c8e-a859-a3f141974026"}
{"level":"info","ts":"2026-11-03T15:29:37.309Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"5925b5bf-1473-4c8e-a859-a3f141974026","1 head service found":"delete-head-after-submit-gg2wh-head-svc"}
{"level":"info","ts":"2026-11-03T15:29:37.309Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"5925b5bf-1473-4c8e-a859-a3f141974026","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-11-03T15:29:37.309Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"5925b5bf-1473-4c8e-a859-a3f141974026","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:29:37.309Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"5925b5bf-1473-4c8e-a859-a3f141974026","worker Pod":"delete-head-after-submit-gg2wh-small-group-worker-dxpfb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod delete-head-after-submit-gg2wh-small-group-worker-dxpfb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:29:37.309Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"5925b5bf-1473-4c8e-a859-a3f141974026","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:29:37.309Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"5925b5bf-1473-4c8e-a859-a3f141974026","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:29:37.309Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"5925b5bf-1473-4c8e-a859-a3f141974026","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:29:37.309Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"5925b5bf-1473-4c8e-a859-a3f141974026","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:29:37.309Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"5925b5bf-1473-4c8e-a859-a3f141974026","podName":"delete-head-after-submit-gg2wh-head-b99bx","pod Status.Phase":"Failed"}
{"level":"info","ts":"2026-11-03T15:29:37.309Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"5925b5bf-1473-4c8e-a859-a3f141974026","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:29:37.309Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"5925b5bf-1473-4c8e-a859-a3f141974026","seconds":300}
{"level":"info","ts":"2026-11-03T15:29:37.573Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"9c961115-5617-4367-9412-8201488b13db"}
{"level":"info","ts":"2026-11-03T15:29:37.573Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"9c961115-5617-4367-9412-8201488b13db"}
{"level":"info","ts":"2026-11-03T15:29:37.573Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"9c961115-5617-4367-9412-8201488b13db","1 head service found":"long-running-hvml9-head-svc"}
{"level":"info","ts":"2026-11-03T15:29:37.573Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"9c961115-5617-4367-9412-8201488b13db","Found 1 head Pod":"long-running-hvml9-head-s8j5v","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:29:37.573Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"9c961115-5617-4367-9412-8201488b13db","head Pod":"long-running-hvml9-head-s8j5v","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-hvml9-head-s8j5v. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:29:37.573Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"9c961115-5617-4367-9412-8201488b13db","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:29:37.573Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"9c961115-5617-4367-9412-8201488b13db","worker Pod":"long-running-hvml9-small-group-worker-mjl4n","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-hvml9-small-group-worker-mjl4n. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:29:37.573Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"9c961115-5617-4367-9412-8201488b13db","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:29:37.573Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"9c961115-5617-4367-9412-8201488b13db","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:29:37.573Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"9c961115-5617-4367-9412-8201488b13db","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:29:37.573Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"9c961115-5617-4367-9412-8201488b13db","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:29:37.574Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"9c961115-5617-4367-9412-8201488b13db","name":"long-running-hvml9","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:29:26Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.42","serviceIP":"10.132.0.42","podName":"long-running-hvml9-head-s8j5v","serviceName":"long-running-hvml9-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:29:06Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:32Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:29:37Z","stateTransitionTimes":{"ready":"2026-11-03T15:29:37Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.42","serviceIP":"10.132.0.42","podName":"long-running-hvml9-head-s8j5v","serviceName":"long-running-hvml9-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:29:06Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:29:37Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:31Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:32Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:29:37.608Z","logger":"controllers.RayClusterMTLS","msg":"Updating head certificate","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"342c0bf1-a19f-44cd-8a07-79807bfaf77a","certificate":"ray-head-cert-delete-head-after-submit-gg2wh","ipAddresses":["127.0.0.1"]}
{"level":"info","ts":"2026-11-03T15:29:37.682Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"7d53d0b7-4524-45e6-9c7a-371ef0e008bd"}
{"level":"info","ts":"2026-11-03T15:29:37.682Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"7d53d0b7-4524-45e6-9c7a-371ef0e008bd"}
{"level":"info","ts":"2026-11-03T15:29:37.682Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"7d53d0b7-4524-45e6-9c7a-371ef0e008bd","1 head service found":"delete-head-after-submit-gg2wh-head-svc"}
{"level":"info","ts":"2026-11-03T15:29:37.682Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"7d53d0b7-4524-45e6-9c7a-371ef0e008bd","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-11-03T15:29:37.682Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"7d53d0b7-4524-45e6-9c7a-371ef0e008bd","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:29:37.682Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"7d53d0b7-4524-45e6-9c7a-371ef0e008bd","worker Pod":"delete-head-after-submit-gg2wh-small-group-worker-dxpfb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod delete-head-after-submit-gg2wh-small-group-worker-dxpfb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:29:37.682Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"7d53d0b7-4524-45e6-9c7a-371ef0e008bd","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:29:37.682Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"7d53d0b7-4524-45e6-9c7a-371ef0e008bd","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:29:37.682Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"7d53d0b7-4524-45e6-9c7a-371ef0e008bd","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:29:37.682Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"7d53d0b7-4524-45e6-9c7a-371ef0e008bd","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:29:37.682Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"7d53d0b7-4524-45e6-9c7a-371ef0e008bd","podName":"delete-head-after-submit-gg2wh-head-b99bx","pod Status.Phase":"Failed"}
{"level":"info","ts":"2026-11-03T15:29:37.682Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"7d53d0b7-4524-45e6-9c7a-371ef0e008bd","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:29:37.682Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"7d53d0b7-4524-45e6-9c7a-371ef0e008bd","seconds":300}
{"level":"info","ts":"2026-11-03T15:29:37.685Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"1ee48411-b80f-4ee4-82be-4a52d2c862e8"}
{"level":"info","ts":"2026-11-03T15:29:37.685Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"1ee48411-b80f-4ee4-82be-4a52d2c862e8"}
{"level":"info","ts":"2026-11-03T15:29:37.685Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"1ee48411-b80f-4ee4-82be-4a52d2c862e8","1 head service found":"delete-head-after-submit-gg2wh-head-svc"}
{"level":"info","ts":"2026-11-03T15:29:37.685Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"1ee48411-b80f-4ee4-82be-4a52d2c862e8"}
{"level":"info","ts":"2026-11-03T15:29:37.790Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"1ee48411-b80f-4ee4-82be-4a52d2c862e8","mode":"IntegratedOAuth","cluster":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:29:37.790Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"1ee48411-b80f-4ee4-82be-4a52d2c862e8","cluster":"delete-head-after-submit-gg2wh","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-11-03T15:29:37.790Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"1ee48411-b80f-4ee4-82be-4a52d2c862e8","cluster":"delete-head-after-submit-gg2wh","authType":"IntegratedOAuth","serviceAccount":"delete-head-after-submit-gg2wh-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-11-03T15:29:37.790Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"1ee48411-b80f-4ee4-82be-4a52d2c862e8","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"delete-head-after-submit-gg2wh","ray.io/group":"headgroup","ray.io/identifier":"delete-head-after-submit-gg2wh-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-11-03T15:29:37.790Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"1ee48411-b80f-4ee4-82be-4a52d2c862e8","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-11-03T15:29:37.790Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"1ee48411-b80f-4ee4-82be-4a52d2c862e8","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-11-03T15:29:37.790Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"1ee48411-b80f-4ee4-82be-4a52d2c862e8","rayNodeType":"head","generatedCmd":"ulimit -n ${RAY_START_ULIMIT_OPEN_FILES:-65536}; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-11-03T15:29:37.790Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"1ee48411-b80f-4ee4-82be-4a52d2c862e8","enabled":true}
{"level":"info","ts":"2026-11-03T15:29:37.852Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"1ee48411-b80f-4ee4-82be-4a52d2c862e8","name":"delete-head-after-submit-gg2wh-head-mscjq"}
{"level":"info","ts":"2026-11-03T15:29:37.852Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"1ee48411-b80f-4ee4-82be-4a52d2c862e8","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:29:37.852Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"1ee48411-b80f-4ee4-82be-4a52d2c862e8","worker Pod":"delete-head-after-submit-gg2wh-small-group-worker-dxpfb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod delete-head-after-submit-gg2wh-small-group-worker-dxpfb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:29:37.852Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"1ee48411-b80f-4ee4-82be-4a52d2c862e8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:29:37.852Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"1ee48411-b80f-4ee4-82be-4a52d2c862e8","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:29:37.852Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"1ee48411-b80f-4ee4-82be-4a52d2c862e8","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:29:37.852Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"1ee48411-b80f-4ee4-82be-4a52d2c862e8","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:29:37.852Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"1ee48411-b80f-4ee4-82be-4a52d2c862e8","podName":"delete-head-after-submit-gg2wh-head-mscjq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:29:37.852Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"1ee48411-b80f-4ee4-82be-4a52d2c862e8","name":"delete-head-after-submit-gg2wh","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:29:37Z","stateTransitionTimes":{"ready":"2026-11-03T15:29:33Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:29:37Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:29:33Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:29:37Z","stateTransitionTimes":{"ready":"2026-11-03T15:29:33Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"delete-head-after-submit-gg2wh-head-mscjq","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:29:37Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:29:33Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:29:37.914Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"aec67454-95bf-46fe-963b-0a6325f781cf"}
{"level":"info","ts":"2026-11-03T15:29:37.914Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"aec67454-95bf-46fe-963b-0a6325f781cf"}
{"level":"info","ts":"2026-11-03T15:29:37.914Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"aec67454-95bf-46fe-963b-0a6325f781cf","1 head service found":"delete-head-after-submit-gg2wh-head-svc"}
{"level":"info","ts":"2026-11-03T15:29:37.914Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"aec67454-95bf-46fe-963b-0a6325f781cf","Found 1 head Pod":"delete-head-after-submit-gg2wh-head-mscjq","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:29:37.914Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"aec67454-95bf-46fe-963b-0a6325f781cf","head Pod":"delete-head-after-submit-gg2wh-head-mscjq","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod delete-head-after-submit-gg2wh-head-mscjq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:29:37.914Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"aec67454-95bf-46fe-963b-0a6325f781cf","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:29:37.914Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"aec67454-95bf-46fe-963b-0a6325f781cf","worker Pod":"delete-head-after-submit-gg2wh-small-group-worker-dxpfb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod delete-head-after-submit-gg2wh-small-group-worker-dxpfb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:29:37.914Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"aec67454-95bf-46fe-963b-0a6325f781cf","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:29:37.914Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"aec67454-95bf-46fe-963b-0a6325f781cf","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:29:37.914Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"aec67454-95bf-46fe-963b-0a6325f781cf","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:29:37.914Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"aec67454-95bf-46fe-963b-0a6325f781cf","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:29:37.914Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"aec67454-95bf-46fe-963b-0a6325f781cf","podName":"delete-head-after-submit-gg2wh-head-mscjq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:29:37.914Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"aec67454-95bf-46fe-963b-0a6325f781cf","name":"delete-head-after-submit-gg2wh","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:29:37Z","stateTransitionTimes":{"ready":"2026-11-03T15:29:33Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:29:37Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:29:33Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:29:37Z","stateTransitionTimes":{"ready":"2026-11-03T15:29:33Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"delete-head-after-submit-gg2wh-head-mscjq","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:29:37Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:29:33Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:29:37.970Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"aec67454-95bf-46fe-963b-0a6325f781cf","name":"delete-head-after-submit-gg2wh","error":"Operation cannot be fulfilled on rayclusters.ray.io \"delete-head-after-submit-gg2wh\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-kbx4w","name":"delete-head-after-submit-gg2wh"}}
{"level":"info","ts":"2026-11-03T15:29:37.970Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"aec67454-95bf-46fe-963b-0a6325f781cf"}
{"level":"error","ts":"2026-11-03T15:29:37.970Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"aec67454-95bf-46fe-963b-0a6325f781cf","error":"Operation cannot be fulfilled on rayclusters.ray.io \"delete-head-after-submit-gg2wh\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:29:37.970Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"57c834bf-1cf6-407f-8ec0-a8f1f53be350"}
{"level":"info","ts":"2026-11-03T15:29:37.970Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"57c834bf-1cf6-407f-8ec0-a8f1f53be350"}
{"level":"info","ts":"2026-11-03T15:29:37.970Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"57c834bf-1cf6-407f-8ec0-a8f1f53be350","1 head service found":"delete-head-after-submit-gg2wh-head-svc"}
{"level":"info","ts":"2026-11-03T15:29:37.970Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"57c834bf-1cf6-407f-8ec0-a8f1f53be350","Found 1 head Pod":"delete-head-after-submit-gg2wh-head-mscjq","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:29:37.970Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"57c834bf-1cf6-407f-8ec0-a8f1f53be350","head Pod":"delete-head-after-submit-gg2wh-head-mscjq","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod delete-head-after-submit-gg2wh-head-mscjq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:29:37.970Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"57c834bf-1cf6-407f-8ec0-a8f1f53be350","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:29:37.970Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"57c834bf-1cf6-407f-8ec0-a8f1f53be350","worker Pod":"delete-head-after-submit-gg2wh-small-group-worker-dxpfb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod delete-head-after-submit-gg2wh-small-group-worker-dxpfb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:29:37.970Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"57c834bf-1cf6-407f-8ec0-a8f1f53be350","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:29:37.970Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"57c834bf-1cf6-407f-8ec0-a8f1f53be350","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:29:37.970Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"57c834bf-1cf6-407f-8ec0-a8f1f53be350","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:29:37.970Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"57c834bf-1cf6-407f-8ec0-a8f1f53be350","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:29:37.970Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"57c834bf-1cf6-407f-8ec0-a8f1f53be350","podName":"delete-head-after-submit-gg2wh-head-mscjq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:29:37.970Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"57c834bf-1cf6-407f-8ec0-a8f1f53be350","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:29:37.970Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"57c834bf-1cf6-407f-8ec0-a8f1f53be350","seconds":300}
{"level":"info","ts":"2026-11-03T15:29:37.978Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"e1a391f2-9310-4ba3-8387-df28539745c6"}
{"level":"info","ts":"2026-11-03T15:29:37.978Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"e1a391f2-9310-4ba3-8387-df28539745c6"}
{"level":"info","ts":"2026-11-03T15:29:37.978Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"e1a391f2-9310-4ba3-8387-df28539745c6","1 head service found":"delete-head-after-submit-gg2wh-head-svc"}
{"level":"info","ts":"2026-11-03T15:29:37.978Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"e1a391f2-9310-4ba3-8387-df28539745c6","Found 1 head Pod":"delete-head-after-submit-gg2wh-head-mscjq","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:29:37.978Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"e1a391f2-9310-4ba3-8387-df28539745c6","head Pod":"delete-head-after-submit-gg2wh-head-mscjq","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod delete-head-after-submit-gg2wh-head-mscjq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:29:37.978Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"e1a391f2-9310-4ba3-8387-df28539745c6","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:29:37.978Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"e1a391f2-9310-4ba3-8387-df28539745c6","worker Pod":"delete-head-after-submit-gg2wh-small-group-worker-dxpfb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod delete-head-after-submit-gg2wh-small-group-worker-dxpfb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:29:37.978Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"e1a391f2-9310-4ba3-8387-df28539745c6","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:29:37.978Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"e1a391f2-9310-4ba3-8387-df28539745c6","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:29:37.978Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"e1a391f2-9310-4ba3-8387-df28539745c6","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:29:37.978Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"e1a391f2-9310-4ba3-8387-df28539745c6","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:29:37.978Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"e1a391f2-9310-4ba3-8387-df28539745c6","podName":"delete-head-after-submit-gg2wh-head-mscjq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:29:37.979Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"e1a391f2-9310-4ba3-8387-df28539745c6","name":"delete-head-after-submit-gg2wh","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:29:37Z","stateTransitionTimes":{"ready":"2026-11-03T15:29:33Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"delete-head-after-submit-gg2wh-head-mscjq","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:29:37Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:29:33Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:29:37Z","stateTransitionTimes":{"ready":"2026-11-03T15:29:33Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"delete-head-after-submit-gg2wh-head-mscjq","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:29:37Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:29:33Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:29:38.157Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"342c0bf1-a19f-44cd-8a07-79807bfaf77a","certificate":"ray-head-cert-delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:29:38.157Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"342c0bf1-a19f-44cd-8a07-79807bfaf77a"}
{"level":"info","ts":"2026-11-03T15:29:38.158Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"489a3fb7-257d-4174-9420-bd926b278f5e","cluster":"long-running-hvml9"}
{"level":"info","ts":"2026-11-03T15:29:38.609Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"9af2c5c6-0847-4728-a6dd-1d9696839cb3"}
{"level":"info","ts":"2026-11-03T15:29:38.609Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"9af2c5c6-0847-4728-a6dd-1d9696839cb3"}
{"level":"info","ts":"2026-11-03T15:29:38.609Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"9af2c5c6-0847-4728-a6dd-1d9696839cb3","1 head service found":"delete-head-after-submit-gg2wh-head-svc"}
{"level":"info","ts":"2026-11-03T15:29:38.609Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"9af2c5c6-0847-4728-a6dd-1d9696839cb3","Found 1 head Pod":"delete-head-after-submit-gg2wh-head-mscjq","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:29:38.609Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"9af2c5c6-0847-4728-a6dd-1d9696839cb3","head Pod":"delete-head-after-submit-gg2wh-head-mscjq","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod delete-head-after-submit-gg2wh-head-mscjq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:29:38.609Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"9af2c5c6-0847-4728-a6dd-1d9696839cb3","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:29:38.609Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"9af2c5c6-0847-4728-a6dd-1d9696839cb3","worker Pod":"delete-head-after-submit-gg2wh-small-group-worker-dxpfb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod delete-head-after-submit-gg2wh-small-group-worker-dxpfb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:29:38.609Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"9af2c5c6-0847-4728-a6dd-1d9696839cb3","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:29:38.609Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"9af2c5c6-0847-4728-a6dd-1d9696839cb3","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:29:38.609Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"9af2c5c6-0847-4728-a6dd-1d9696839cb3","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:29:38.609Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"9af2c5c6-0847-4728-a6dd-1d9696839cb3","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:29:38.609Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"9af2c5c6-0847-4728-a6dd-1d9696839cb3","podName":"delete-head-after-submit-gg2wh-head-mscjq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:29:38.609Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"9af2c5c6-0847-4728-a6dd-1d9696839cb3","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:29:38.609Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"9af2c5c6-0847-4728-a6dd-1d9696839cb3","seconds":300}
{"level":"info","ts":"2026-11-03T15:29:38.973Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"9bd1972c-2e7d-4b00-8bc6-83497e41afa7","cluster":"delete-head-after-submit-gg2wh"}
{"level":"error","ts":"2026-11-03T15:29:39.303Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"af7f7255-ed01-4e71-b945-0e1b2e5b55f2","JobId":"delete-head-after-submit-cwfp8","error":"Get \"http://delete-head-after-submit-gg2wh-head-svc.test-ns-kbx4w.svc.cluster.local:8265/api/jobs/delete-head-after-submit-cwfp8\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:321\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:29:39.304Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"47e5bbd2-14fb-4ac8-9905-7d694ab7f080","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:29:39.304Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"47e5bbd2-14fb-4ac8-9905-7d694ab7f080","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Failed","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":600,"Status.endTime":"2026-11-03 15:27:38 +0000 UTC","Now":"2026-11-03T15:29:39.304Z","ShutdownTime":"2026-11-03T15:37:38.000Z"}
{"level":"info","ts":"2026-11-03T15:29:39.304Z","logger":"controllers.RayJob","msg":"TTL has not been met for ShutdownAfterJobFinishes. Requeuing.","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"47e5bbd2-14fb-4ac8-9905-7d694ab7f080","deletionMechanism":"ShutdownAfterJobFinishes","shutdownTime":"2026-11-03T15:37:38.000Z","requeueAfter":480.695849195}
{"level":"info","ts":"2026-11-03T15:29:39.304Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"be0daf20-fbb0-42fa-a971-b703d9743d62","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:29:39.304Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"be0daf20-fbb0-42fa-a971-b703d9743d62","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:29:39.304Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"be0daf20-fbb0-42fa-a971-b703d9743d62","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:29:39.304Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"be0daf20-fbb0-42fa-a971-b703d9743d62"}
{"level":"info","ts":"2026-11-03T15:29:39.304Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"be0daf20-fbb0-42fa-a971-b703d9743d62","rayCluster":"delete-head-after-submit-gg2wh","serviceType":"headService"}
{"level":"error","ts":"2026-11-03T15:29:39.306Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"be0daf20-fbb0-42fa-a971-b703d9743d62","JobId":"delete-head-after-submit-cwfp8","error":"Get \"http://delete-head-after-submit-gg2wh-head-svc.test-ns-kbx4w.svc.cluster.local:8265/api/jobs/delete-head-after-submit-cwfp8\": dial tcp: lookup delete-head-after-submit-gg2wh-head-svc.test-ns-kbx4w.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:321\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:29:39.635Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"483cbe4e-4184-4742-9f69-b52b3024747f"}
{"level":"info","ts":"2026-11-03T15:29:39.636Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"483cbe4e-4184-4742-9f69-b52b3024747f"}
{"level":"info","ts":"2026-11-03T15:29:39.636Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"483cbe4e-4184-4742-9f69-b52b3024747f","1 head service found":"long-running-hvml9-head-svc"}
{"level":"info","ts":"2026-11-03T15:29:39.636Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"483cbe4e-4184-4742-9f69-b52b3024747f","Found 1 head Pod":"long-running-hvml9-head-s8j5v","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:29:39.636Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"483cbe4e-4184-4742-9f69-b52b3024747f","head Pod":"long-running-hvml9-head-s8j5v","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-hvml9-head-s8j5v. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:29:39.636Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"483cbe4e-4184-4742-9f69-b52b3024747f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:29:39.636Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"483cbe4e-4184-4742-9f69-b52b3024747f","worker Pod":"long-running-hvml9-small-group-worker-mjl4n","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-hvml9-small-group-worker-mjl4n. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:29:39.636Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"483cbe4e-4184-4742-9f69-b52b3024747f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:29:39.636Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"483cbe4e-4184-4742-9f69-b52b3024747f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:29:39.636Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"483cbe4e-4184-4742-9f69-b52b3024747f","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:29:39.636Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"483cbe4e-4184-4742-9f69-b52b3024747f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:29:39.636Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"483cbe4e-4184-4742-9f69-b52b3024747f","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:29:39.636Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"483cbe4e-4184-4742-9f69-b52b3024747f","seconds":300}
{"level":"info","ts":"2026-11-03T15:29:39.677Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"906b88e3-7fb2-4ca4-bfb1-a84214ee632c"}
{"level":"info","ts":"2026-11-03T15:29:39.677Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"906b88e3-7fb2-4ca4-bfb1-a84214ee632c"}
{"level":"info","ts":"2026-11-03T15:29:39.677Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"906b88e3-7fb2-4ca4-bfb1-a84214ee632c","1 head service found":"delete-head-after-submit-gg2wh-head-svc"}
{"level":"info","ts":"2026-11-03T15:29:39.677Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"906b88e3-7fb2-4ca4-bfb1-a84214ee632c","Found 1 head Pod":"delete-head-after-submit-gg2wh-head-mscjq","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:29:39.677Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"906b88e3-7fb2-4ca4-bfb1-a84214ee632c","head Pod":"delete-head-after-submit-gg2wh-head-mscjq","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod delete-head-after-submit-gg2wh-head-mscjq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:29:39.678Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"906b88e3-7fb2-4ca4-bfb1-a84214ee632c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:29:39.678Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"906b88e3-7fb2-4ca4-bfb1-a84214ee632c","worker Pod":"delete-head-after-submit-gg2wh-small-group-worker-dxpfb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod delete-head-after-submit-gg2wh-small-group-worker-dxpfb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:29:39.678Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"906b88e3-7fb2-4ca4-bfb1-a84214ee632c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:29:39.678Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"906b88e3-7fb2-4ca4-bfb1-a84214ee632c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:29:39.678Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"906b88e3-7fb2-4ca4-bfb1-a84214ee632c","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:29:39.678Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"906b88e3-7fb2-4ca4-bfb1-a84214ee632c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:29:39.678Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"906b88e3-7fb2-4ca4-bfb1-a84214ee632c","podName":"delete-head-after-submit-gg2wh-head-mscjq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:29:39.678Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"906b88e3-7fb2-4ca4-bfb1-a84214ee632c","name":"delete-head-after-submit-gg2wh","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:29:37Z","stateTransitionTimes":{"ready":"2026-11-03T15:29:33Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"delete-head-after-submit-gg2wh-head-mscjq","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:29:37Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:29:33Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:29:39Z","stateTransitionTimes":{"ready":"2026-11-03T15:29:33Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.49","serviceIP":"10.134.0.49","podName":"delete-head-after-submit-gg2wh-head-mscjq","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:29:37Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:29:33Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:29:39.739Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"a6e7bb47-d9ba-4e72-86b8-1deba0130fac","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:29:39.739Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"a6e7bb47-d9ba-4e72-86b8-1deba0130fac","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:29:39.739Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"a6e7bb47-d9ba-4e72-86b8-1deba0130fac","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:29:39.739Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"a6e7bb47-d9ba-4e72-86b8-1deba0130fac"}
{"level":"info","ts":"2026-11-03T15:29:39.739Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"a6e7bb47-d9ba-4e72-86b8-1deba0130fac","rayCluster":"delete-head-after-submit-gg2wh","serviceType":"headService"}
{"level":"error","ts":"2026-11-03T15:29:39.746Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"a6e7bb47-d9ba-4e72-86b8-1deba0130fac","JobId":"delete-head-after-submit-cwfp8","error":"Get \"http://delete-head-after-submit-gg2wh-head-svc.test-ns-kbx4w.svc.cluster.local:8265/api/jobs/delete-head-after-submit-cwfp8\": dial tcp: lookup delete-head-after-submit-gg2wh-head-svc.test-ns-kbx4w.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:321\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:29:39.789Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"438fbb22-5297-48e5-9e56-073edd6d2e1b","cluster":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:29:40.094Z","logger":"controllers.RayClusterMTLS","msg":"Updating head certificate","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"438fbb22-5297-48e5-9e56-073edd6d2e1b","certificate":"ray-head-cert-delete-head-after-submit-gg2wh","ipAddresses":["10.134.0.49","127.0.0.1"]}
{"level":"info","ts":"2026-11-03T15:29:40.507Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"438fbb22-5297-48e5-9e56-073edd6d2e1b","certificate":"ray-head-cert-delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:29:40.507Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"438fbb22-5297-48e5-9e56-073edd6d2e1b"}
{"level":"info","ts":"2026-11-03T15:29:41.739Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"4e061597-408f-4207-a3ab-8000c25d3dc4"}
{"level":"info","ts":"2026-11-03T15:29:41.739Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"4e061597-408f-4207-a3ab-8000c25d3dc4"}
{"level":"info","ts":"2026-11-03T15:29:41.739Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"4e061597-408f-4207-a3ab-8000c25d3dc4","1 head service found":"delete-head-after-submit-gg2wh-head-svc"}
{"level":"info","ts":"2026-11-03T15:29:41.739Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"4e061597-408f-4207-a3ab-8000c25d3dc4","Found 1 head Pod":"delete-head-after-submit-gg2wh-head-mscjq","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:29:41.739Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"4e061597-408f-4207-a3ab-8000c25d3dc4","head Pod":"delete-head-after-submit-gg2wh-head-mscjq","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod delete-head-after-submit-gg2wh-head-mscjq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:29:41.739Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"4e061597-408f-4207-a3ab-8000c25d3dc4","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:29:41.739Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"4e061597-408f-4207-a3ab-8000c25d3dc4","worker Pod":"delete-head-after-submit-gg2wh-small-group-worker-dxpfb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod delete-head-after-submit-gg2wh-small-group-worker-dxpfb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:29:41.739Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"4e061597-408f-4207-a3ab-8000c25d3dc4","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:29:41.739Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"4e061597-408f-4207-a3ab-8000c25d3dc4","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:29:41.739Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"4e061597-408f-4207-a3ab-8000c25d3dc4","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:29:41.739Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"4e061597-408f-4207-a3ab-8000c25d3dc4","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:29:41.739Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"4e061597-408f-4207-a3ab-8000c25d3dc4","podName":"delete-head-after-submit-gg2wh-head-mscjq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:29:41.739Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"4e061597-408f-4207-a3ab-8000c25d3dc4","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:29:41.739Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"4e061597-408f-4207-a3ab-8000c25d3dc4","seconds":300}
{"level":"info","ts":"2026-11-03T15:29:42.746Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"8999158f-df09-4295-bb63-2f800773161d","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:29:42.746Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"8999158f-df09-4295-bb63-2f800773161d","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:29:42.746Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"8999158f-df09-4295-bb63-2f800773161d","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:29:42.747Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"8999158f-df09-4295-bb63-2f800773161d"}
{"level":"info","ts":"2026-11-03T15:29:42.747Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"8999158f-df09-4295-bb63-2f800773161d","rayCluster":"delete-head-after-submit-gg2wh","serviceType":"headService"}
{"level":"error","ts":"2026-11-03T15:29:42.749Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"8999158f-df09-4295-bb63-2f800773161d","JobId":"delete-head-after-submit-cwfp8","error":"Get \"http://delete-head-after-submit-gg2wh-head-svc.test-ns-kbx4w.svc.cluster.local:8265/api/jobs/delete-head-after-submit-cwfp8\": dial tcp: lookup delete-head-after-submit-gg2wh-head-svc.test-ns-kbx4w.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:321\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:29:45.749Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"fc82220f-12e4-4150-a5d1-ab466b7da720","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:29:45.749Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"fc82220f-12e4-4150-a5d1-ab466b7da720","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:29:45.749Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"fc82220f-12e4-4150-a5d1-ab466b7da720","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:29:45.749Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"fc82220f-12e4-4150-a5d1-ab466b7da720"}
{"level":"info","ts":"2026-11-03T15:29:45.749Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"fc82220f-12e4-4150-a5d1-ab466b7da720","rayCluster":"delete-head-after-submit-gg2wh","serviceType":"headService"}
{"level":"error","ts":"2026-11-03T15:29:45.753Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"fc82220f-12e4-4150-a5d1-ab466b7da720","JobId":"delete-head-after-submit-cwfp8","error":"Get \"http://delete-head-after-submit-gg2wh-head-svc.test-ns-kbx4w.svc.cluster.local:8265/api/jobs/delete-head-after-submit-cwfp8\": dial tcp 10.134.0.49:8265: connect: connection refused","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:321\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:29:48.754Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"54102400-021a-4595-ab89-f5615b8e8b38","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:29:48.754Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"54102400-021a-4595-ab89-f5615b8e8b38","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:29:48.754Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"54102400-021a-4595-ab89-f5615b8e8b38","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:29:48.754Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"54102400-021a-4595-ab89-f5615b8e8b38"}
{"level":"info","ts":"2026-11-03T15:29:48.754Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"54102400-021a-4595-ab89-f5615b8e8b38","rayCluster":"delete-head-after-submit-gg2wh","serviceType":"headService"}
{"level":"error","ts":"2026-11-03T15:29:48.756Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"54102400-021a-4595-ab89-f5615b8e8b38","JobId":"delete-head-after-submit-cwfp8","error":"Get \"http://delete-head-after-submit-gg2wh-head-svc.test-ns-kbx4w.svc.cluster.local:8265/api/jobs/delete-head-after-submit-cwfp8\": dial tcp 10.134.0.49:8265: connect: connection refused","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:321\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:29:51.757Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"06405f00-8fec-4eed-a657-6d1adbd771e2","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:29:51.757Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"06405f00-8fec-4eed-a657-6d1adbd771e2","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:29:51.757Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"06405f00-8fec-4eed-a657-6d1adbd771e2","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:29:51.757Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"06405f00-8fec-4eed-a657-6d1adbd771e2"}
{"level":"info","ts":"2026-11-03T15:29:51.757Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"06405f00-8fec-4eed-a657-6d1adbd771e2","rayCluster":"delete-head-after-submit-gg2wh","serviceType":"headService"}
{"level":"error","ts":"2026-11-03T15:29:51.760Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"06405f00-8fec-4eed-a657-6d1adbd771e2","JobId":"delete-head-after-submit-cwfp8","error":"Get \"http://delete-head-after-submit-gg2wh-head-svc.test-ns-kbx4w.svc.cluster.local:8265/api/jobs/delete-head-after-submit-cwfp8\": dial tcp 10.134.0.49:8265: connect: connection refused","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:321\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:29:54.761Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"408bcfad-505e-48ed-9f82-933731830452","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:29:54.761Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"408bcfad-505e-48ed-9f82-933731830452","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:29:54.761Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"408bcfad-505e-48ed-9f82-933731830452","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:29:54.761Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"408bcfad-505e-48ed-9f82-933731830452"}
{"level":"info","ts":"2026-11-03T15:29:54.761Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"408bcfad-505e-48ed-9f82-933731830452","rayCluster":"delete-head-after-submit-gg2wh","serviceType":"headService"}
{"level":"error","ts":"2026-11-03T15:29:54.764Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"408bcfad-505e-48ed-9f82-933731830452","JobId":"delete-head-after-submit-cwfp8","error":"Get \"http://delete-head-after-submit-gg2wh-head-svc.test-ns-kbx4w.svc.cluster.local:8265/api/jobs/delete-head-after-submit-cwfp8\": dial tcp 10.134.0.49:8265: connect: connection refused","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:321\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:29:57.765Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"067638a3-cae5-43d2-a496-7da08e4925e1","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:29:57.765Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"067638a3-cae5-43d2-a496-7da08e4925e1","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:29:57.765Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"067638a3-cae5-43d2-a496-7da08e4925e1","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:29:57.765Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"067638a3-cae5-43d2-a496-7da08e4925e1"}
{"level":"info","ts":"2026-11-03T15:29:57.765Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"067638a3-cae5-43d2-a496-7da08e4925e1","rayCluster":"delete-head-after-submit-gg2wh","serviceType":"headService"}
{"level":"error","ts":"2026-11-03T15:29:57.767Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"067638a3-cae5-43d2-a496-7da08e4925e1","JobId":"delete-head-after-submit-cwfp8","error":"Get \"http://delete-head-after-submit-gg2wh-head-svc.test-ns-kbx4w.svc.cluster.local:8265/api/jobs/delete-head-after-submit-cwfp8\": dial tcp 10.134.0.49:8265: connect: connection refused","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:321\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:30:00.768Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"50662974-f36a-4114-8134-73b1a3a5e989","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:30:00.768Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"50662974-f36a-4114-8134-73b1a3a5e989","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:30:00.768Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"50662974-f36a-4114-8134-73b1a3a5e989","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:30:00.768Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"50662974-f36a-4114-8134-73b1a3a5e989"}
{"level":"info","ts":"2026-11-03T15:30:00.768Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"50662974-f36a-4114-8134-73b1a3a5e989","rayCluster":"delete-head-after-submit-gg2wh","serviceType":"headService"}
{"level":"error","ts":"2026-11-03T15:30:00.771Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"50662974-f36a-4114-8134-73b1a3a5e989","JobId":"delete-head-after-submit-cwfp8","error":"Get \"http://delete-head-after-submit-gg2wh-head-svc.test-ns-kbx4w.svc.cluster.local:8265/api/jobs/delete-head-after-submit-cwfp8\": dial tcp 10.134.0.49:8265: connect: connection refused","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:321\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:30:03.772Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"3ff53e5e-7f6e-40fc-9529-1c0ef339a7f8","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:30:03.772Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"3ff53e5e-7f6e-40fc-9529-1c0ef339a7f8","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:30:03.772Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"3ff53e5e-7f6e-40fc-9529-1c0ef339a7f8","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:30:03.772Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"3ff53e5e-7f6e-40fc-9529-1c0ef339a7f8"}
{"level":"info","ts":"2026-11-03T15:30:03.772Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"3ff53e5e-7f6e-40fc-9529-1c0ef339a7f8","rayCluster":"delete-head-after-submit-gg2wh","serviceType":"headService"}
{"level":"error","ts":"2026-11-03T15:30:03.775Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"3ff53e5e-7f6e-40fc-9529-1c0ef339a7f8","JobId":"delete-head-after-submit-cwfp8","error":"Get \"http://delete-head-after-submit-gg2wh-head-svc.test-ns-kbx4w.svc.cluster.local:8265/api/jobs/delete-head-after-submit-cwfp8\": dial tcp 10.134.0.49:8265: connect: connection refused","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:321\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:30:06.776Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"b0a6662c-a3f0-4494-8cba-dabdc8d0aaf8","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:30:06.776Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"b0a6662c-a3f0-4494-8cba-dabdc8d0aaf8","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:30:06.776Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"b0a6662c-a3f0-4494-8cba-dabdc8d0aaf8","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:30:06.776Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"b0a6662c-a3f0-4494-8cba-dabdc8d0aaf8"}
{"level":"info","ts":"2026-11-03T15:30:06.776Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"b0a6662c-a3f0-4494-8cba-dabdc8d0aaf8","rayCluster":"delete-head-after-submit-gg2wh","serviceType":"headService"}
{"level":"error","ts":"2026-11-03T15:30:06.781Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"b0a6662c-a3f0-4494-8cba-dabdc8d0aaf8","JobId":"delete-head-after-submit-cwfp8","error":"Get \"http://delete-head-after-submit-gg2wh-head-svc.test-ns-kbx4w.svc.cluster.local:8265/api/jobs/delete-head-after-submit-cwfp8\": dial tcp 10.134.0.49:8265: connect: connection refused","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:321\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:30:09.782Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"5bcac8bd-67a8-48ab-b146-73a9784a28de","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:30:09.782Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"5bcac8bd-67a8-48ab-b146-73a9784a28de","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:30:09.782Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"5bcac8bd-67a8-48ab-b146-73a9784a28de","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:30:09.782Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"5bcac8bd-67a8-48ab-b146-73a9784a28de"}
{"level":"info","ts":"2026-11-03T15:30:09.782Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"5bcac8bd-67a8-48ab-b146-73a9784a28de","rayCluster":"delete-head-after-submit-gg2wh","serviceType":"headService"}
{"level":"error","ts":"2026-11-03T15:30:09.785Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"5bcac8bd-67a8-48ab-b146-73a9784a28de","JobId":"delete-head-after-submit-cwfp8","error":"Get \"http://delete-head-after-submit-gg2wh-head-svc.test-ns-kbx4w.svc.cluster.local:8265/api/jobs/delete-head-after-submit-cwfp8\": dial tcp 10.134.0.49:8265: connect: connection refused","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:321\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:30:10.507Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"aa514080-5c18-436c-aebe-c88dbe512367","cluster":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:30:12.785Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"287b2ba9-380a-428e-863e-d392e6bb5b2f","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:30:12.785Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"287b2ba9-380a-428e-863e-d392e6bb5b2f","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:30:12.785Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"287b2ba9-380a-428e-863e-d392e6bb5b2f","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:30:12.785Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"287b2ba9-380a-428e-863e-d392e6bb5b2f"}
{"level":"info","ts":"2026-11-03T15:30:12.785Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"287b2ba9-380a-428e-863e-d392e6bb5b2f","rayCluster":"delete-head-after-submit-gg2wh","serviceType":"headService"}
{"level":"error","ts":"2026-11-03T15:30:12.788Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"287b2ba9-380a-428e-863e-d392e6bb5b2f","JobId":"delete-head-after-submit-cwfp8","error":"Get \"http://delete-head-after-submit-gg2wh-head-svc.test-ns-kbx4w.svc.cluster.local:8265/api/jobs/delete-head-after-submit-cwfp8\": dial tcp 10.134.0.49:8265: connect: connection refused","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:321\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:30:15.789Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"be868756-372c-406c-8cd2-11640439b657","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:30:15.789Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"be868756-372c-406c-8cd2-11640439b657","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:30:15.789Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"be868756-372c-406c-8cd2-11640439b657","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:30:15.789Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"be868756-372c-406c-8cd2-11640439b657"}
{"level":"info","ts":"2026-11-03T15:30:15.789Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"be868756-372c-406c-8cd2-11640439b657","rayCluster":"delete-head-after-submit-gg2wh","serviceType":"headService"}
{"level":"error","ts":"2026-11-03T15:30:15.791Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"be868756-372c-406c-8cd2-11640439b657","JobId":"delete-head-after-submit-cwfp8","error":"Get \"http://delete-head-after-submit-gg2wh-head-svc.test-ns-kbx4w.svc.cluster.local:8265/api/jobs/delete-head-after-submit-cwfp8\": dial tcp 10.134.0.49:8265: connect: connection refused","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:321\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:30:18.793Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"3e09fdf9-2874-478f-9f24-1e452d7d86de","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:30:18.793Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"3e09fdf9-2874-478f-9f24-1e452d7d86de","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:30:18.793Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"3e09fdf9-2874-478f-9f24-1e452d7d86de","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:30:18.793Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"3e09fdf9-2874-478f-9f24-1e452d7d86de"}
{"level":"info","ts":"2026-11-03T15:30:18.793Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"3e09fdf9-2874-478f-9f24-1e452d7d86de","rayCluster":"delete-head-after-submit-gg2wh","serviceType":"headService"}
{"level":"error","ts":"2026-11-03T15:30:18.795Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"3e09fdf9-2874-478f-9f24-1e452d7d86de","JobId":"delete-head-after-submit-cwfp8","error":"Get \"http://delete-head-after-submit-gg2wh-head-svc.test-ns-kbx4w.svc.cluster.local:8265/api/jobs/delete-head-after-submit-cwfp8\": dial tcp 10.134.0.49:8265: connect: connection refused","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:321\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:30:21.796Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"adf93ffd-d238-494a-b1bb-eefc6bb5036f","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:30:21.796Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"adf93ffd-d238-494a-b1bb-eefc6bb5036f","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:30:21.796Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"adf93ffd-d238-494a-b1bb-eefc6bb5036f","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:30:21.796Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"adf93ffd-d238-494a-b1bb-eefc6bb5036f"}
{"level":"info","ts":"2026-11-03T15:30:21.796Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"adf93ffd-d238-494a-b1bb-eefc6bb5036f","rayCluster":"delete-head-after-submit-gg2wh","serviceType":"headService"}
{"level":"error","ts":"2026-11-03T15:30:21.800Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"adf93ffd-d238-494a-b1bb-eefc6bb5036f","JobId":"delete-head-after-submit-cwfp8","error":"Get \"http://delete-head-after-submit-gg2wh-head-svc.test-ns-kbx4w.svc.cluster.local:8265/api/jobs/delete-head-after-submit-cwfp8\": dial tcp 10.134.0.49:8265: connect: connection refused","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:321\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:30:24.800Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"162d9875-6694-4c2a-a611-078f91995aaa","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:30:24.800Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"162d9875-6694-4c2a-a611-078f91995aaa","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:30:24.800Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"162d9875-6694-4c2a-a611-078f91995aaa","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:30:24.801Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"162d9875-6694-4c2a-a611-078f91995aaa"}
{"level":"info","ts":"2026-11-03T15:30:24.801Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"162d9875-6694-4c2a-a611-078f91995aaa","rayCluster":"delete-head-after-submit-gg2wh","serviceType":"headService"}
{"level":"error","ts":"2026-11-03T15:30:24.803Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"162d9875-6694-4c2a-a611-078f91995aaa","JobId":"delete-head-after-submit-cwfp8","error":"Get \"http://delete-head-after-submit-gg2wh-head-svc.test-ns-kbx4w.svc.cluster.local:8265/api/jobs/delete-head-after-submit-cwfp8\": dial tcp 10.134.0.49:8265: connect: connection refused","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:321\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:30:27.804Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"490a4d01-b796-4621-9eeb-c06b6b14058a","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:30:27.804Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"490a4d01-b796-4621-9eeb-c06b6b14058a","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:30:27.804Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"490a4d01-b796-4621-9eeb-c06b6b14058a","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:30:27.804Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"490a4d01-b796-4621-9eeb-c06b6b14058a"}
{"level":"info","ts":"2026-11-03T15:30:27.804Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"490a4d01-b796-4621-9eeb-c06b6b14058a","rayCluster":"delete-head-after-submit-gg2wh","serviceType":"headService"}
{"level":"error","ts":"2026-11-03T15:30:27.807Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"490a4d01-b796-4621-9eeb-c06b6b14058a","JobId":"delete-head-after-submit-cwfp8","error":"Get \"http://delete-head-after-submit-gg2wh-head-svc.test-ns-kbx4w.svc.cluster.local:8265/api/jobs/delete-head-after-submit-cwfp8\": dial tcp 10.134.0.49:8265: connect: connection refused","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:321\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:30:30.808Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"d795e1d8-5577-4aac-bf59-9ca357f137c3","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:30:30.808Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"d795e1d8-5577-4aac-bf59-9ca357f137c3","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:30:30.808Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"d795e1d8-5577-4aac-bf59-9ca357f137c3","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:30:30.808Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"d795e1d8-5577-4aac-bf59-9ca357f137c3"}
{"level":"info","ts":"2026-11-03T15:30:30.808Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"d795e1d8-5577-4aac-bf59-9ca357f137c3","rayCluster":"delete-head-after-submit-gg2wh","serviceType":"headService"}
{"level":"error","ts":"2026-11-03T15:30:30.811Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"d795e1d8-5577-4aac-bf59-9ca357f137c3","JobId":"delete-head-after-submit-cwfp8","error":"Get \"http://delete-head-after-submit-gg2wh-head-svc.test-ns-kbx4w.svc.cluster.local:8265/api/jobs/delete-head-after-submit-cwfp8\": dial tcp 10.134.0.49:8265: connect: connection refused","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:321\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:30:33.811Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"29291ee5-551c-4a13-a302-4d21e602b641","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:30:33.811Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"29291ee5-551c-4a13-a302-4d21e602b641","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:30:33.811Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"29291ee5-551c-4a13-a302-4d21e602b641","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:30:33.811Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"29291ee5-551c-4a13-a302-4d21e602b641"}
{"level":"info","ts":"2026-11-03T15:30:33.811Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"29291ee5-551c-4a13-a302-4d21e602b641","rayCluster":"delete-head-after-submit-gg2wh","serviceType":"headService"}
{"level":"error","ts":"2026-11-03T15:30:33.813Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"29291ee5-551c-4a13-a302-4d21e602b641","JobId":"delete-head-after-submit-cwfp8","error":"Get \"http://delete-head-after-submit-gg2wh-head-svc.test-ns-kbx4w.svc.cluster.local:8265/api/jobs/delete-head-after-submit-cwfp8\": dial tcp 10.134.0.49:8265: connect: connection refused","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:321\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:30:36.814Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"b90d793d-7d7b-4403-8086-4067afe33981","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:30:36.814Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"b90d793d-7d7b-4403-8086-4067afe33981","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:30:36.814Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"b90d793d-7d7b-4403-8086-4067afe33981","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:30:36.814Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"b90d793d-7d7b-4403-8086-4067afe33981"}
{"level":"info","ts":"2026-11-03T15:30:36.814Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"b90d793d-7d7b-4403-8086-4067afe33981","rayCluster":"delete-head-after-submit-gg2wh","serviceType":"headService"}
{"level":"error","ts":"2026-11-03T15:30:36.820Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"b90d793d-7d7b-4403-8086-4067afe33981","JobId":"delete-head-after-submit-cwfp8","error":"Get \"http://delete-head-after-submit-gg2wh-head-svc.test-ns-kbx4w.svc.cluster.local:8265/api/jobs/delete-head-after-submit-cwfp8\": dial tcp 10.134.0.49:8265: connect: connection refused","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:321\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:30:37.786Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"3bb34bdb-24f1-49d2-954d-3a48a68c30af"}
{"level":"info","ts":"2026-11-03T15:30:37.786Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"3bb34bdb-24f1-49d2-954d-3a48a68c30af"}
{"level":"info","ts":"2026-11-03T15:30:37.786Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"3bb34bdb-24f1-49d2-954d-3a48a68c30af","1 head service found":"delete-head-after-submit-gg2wh-head-svc"}
{"level":"info","ts":"2026-11-03T15:30:37.786Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"3bb34bdb-24f1-49d2-954d-3a48a68c30af","Found 1 head Pod":"delete-head-after-submit-gg2wh-head-mscjq","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:30:37.786Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"3bb34bdb-24f1-49d2-954d-3a48a68c30af","head Pod":"delete-head-after-submit-gg2wh-head-mscjq","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod delete-head-after-submit-gg2wh-head-mscjq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:30:37.786Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"3bb34bdb-24f1-49d2-954d-3a48a68c30af","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:30:37.786Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"3bb34bdb-24f1-49d2-954d-3a48a68c30af","worker Pod":"delete-head-after-submit-gg2wh-small-group-worker-dxpfb","shouldDelete":false,"reason":"The Pod status of the worker Pod delete-head-after-submit-gg2wh-small-group-worker-dxpfb is Running, and the Ray container terminated status is &ContainerStateTerminated{ExitCode:1,Signal:0,Reason:Error,Message:,StartedAt:2026-11-03 15:29:21 +0000 UTC,FinishedAt:2026-11-03 15:30:37 +0000 UTC,ContainerID:cri-o://334d5685c78374951167443028518f9217618b7ff79816f4797c5fb30df8dffb,}. However, KubeRay will not delete the Pod because its restartPolicy is set to Always and it should be able to restart automatically."}
{"level":"info","ts":"2026-11-03T15:30:37.786Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"3bb34bdb-24f1-49d2-954d-3a48a68c30af","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:30:37.786Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"3bb34bdb-24f1-49d2-954d-3a48a68c30af","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:30:37.786Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"3bb34bdb-24f1-49d2-954d-3a48a68c30af","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:30:37.786Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"3bb34bdb-24f1-49d2-954d-3a48a68c30af","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:30:37.786Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"3bb34bdb-24f1-49d2-954d-3a48a68c30af","podName":"delete-head-after-submit-gg2wh-head-mscjq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:30:37.786Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"3bb34bdb-24f1-49d2-954d-3a48a68c30af","name":"delete-head-after-submit-gg2wh","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:29:39Z","stateTransitionTimes":{"ready":"2026-11-03T15:29:33Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.49","serviceIP":"10.134.0.49","podName":"delete-head-after-submit-gg2wh-head-mscjq","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:29:37Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:29:33Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:30:37Z","stateTransitionTimes":{"ready":"2026-11-03T15:29:33Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.49","serviceIP":"10.134.0.49","podName":"delete-head-after-submit-gg2wh-head-mscjq","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:29:37Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:29:33Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:30:37.850Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"a7f1c547-e502-4046-ab61-f105fa94961c","cluster":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:30:37.850Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"6e24ce5f-a267-4142-bef9-dbb84f077f37","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:30:37.850Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"6e24ce5f-a267-4142-bef9-dbb84f077f37","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:30:37.850Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"6e24ce5f-a267-4142-bef9-dbb84f077f37","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:30:37.850Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"6e24ce5f-a267-4142-bef9-dbb84f077f37"}
{"level":"info","ts":"2026-11-03T15:30:37.850Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"6e24ce5f-a267-4142-bef9-dbb84f077f37","rayCluster":"delete-head-after-submit-gg2wh","serviceType":"headService"}
{"level":"error","ts":"2026-11-03T15:30:37.853Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"6e24ce5f-a267-4142-bef9-dbb84f077f37","JobId":"delete-head-after-submit-cwfp8","error":"Get \"http://delete-head-after-submit-gg2wh-head-svc.test-ns-kbx4w.svc.cluster.local:8265/api/jobs/delete-head-after-submit-cwfp8\": dial tcp 10.134.0.49:8265: connect: connection refused","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:321\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:30:38.745Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"7d0134ed-6608-4a6a-8707-f48a3acfd913"}
{"level":"info","ts":"2026-11-03T15:30:38.745Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"7d0134ed-6608-4a6a-8707-f48a3acfd913"}
{"level":"info","ts":"2026-11-03T15:30:38.745Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"7d0134ed-6608-4a6a-8707-f48a3acfd913","1 head service found":"delete-head-after-submit-gg2wh-head-svc"}
{"level":"info","ts":"2026-11-03T15:30:38.745Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"7d0134ed-6608-4a6a-8707-f48a3acfd913","Found 1 head Pod":"delete-head-after-submit-gg2wh-head-mscjq","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:30:38.745Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"7d0134ed-6608-4a6a-8707-f48a3acfd913","head Pod":"delete-head-after-submit-gg2wh-head-mscjq","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod delete-head-after-submit-gg2wh-head-mscjq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:30:38.745Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"7d0134ed-6608-4a6a-8707-f48a3acfd913","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:30:38.745Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"7d0134ed-6608-4a6a-8707-f48a3acfd913","worker Pod":"delete-head-after-submit-gg2wh-small-group-worker-dxpfb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod delete-head-after-submit-gg2wh-small-group-worker-dxpfb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:30:38.745Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"7d0134ed-6608-4a6a-8707-f48a3acfd913","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:30:38.745Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"7d0134ed-6608-4a6a-8707-f48a3acfd913","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:30:38.745Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"7d0134ed-6608-4a6a-8707-f48a3acfd913","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:30:38.745Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"7d0134ed-6608-4a6a-8707-f48a3acfd913","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:30:38.745Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"7d0134ed-6608-4a6a-8707-f48a3acfd913","podName":"delete-head-after-submit-gg2wh-head-mscjq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:30:38.745Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"7d0134ed-6608-4a6a-8707-f48a3acfd913","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:30:38.745Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"7d0134ed-6608-4a6a-8707-f48a3acfd913","seconds":300}
{"level":"info","ts":"2026-11-03T15:30:38.974Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"9153919f-07c0-4afd-b8c4-a71dc0801775","cluster":"long-running-hvml9"}
{"level":"info","ts":"2026-11-03T15:30:40.854Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"9d6ecc89-7619-40b7-99d1-34f09f36b4fe","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:30:40.854Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"9d6ecc89-7619-40b7-99d1-34f09f36b4fe","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:30:40.854Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"9d6ecc89-7619-40b7-99d1-34f09f36b4fe","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:30:40.854Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"9d6ecc89-7619-40b7-99d1-34f09f36b4fe"}
{"level":"info","ts":"2026-11-03T15:30:40.854Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"9d6ecc89-7619-40b7-99d1-34f09f36b4fe","rayCluster":"delete-head-after-submit-gg2wh","serviceType":"headService"}
{"level":"error","ts":"2026-11-03T15:30:40.857Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"9d6ecc89-7619-40b7-99d1-34f09f36b4fe","JobId":"delete-head-after-submit-cwfp8","error":"Get \"http://delete-head-after-submit-gg2wh-head-svc.test-ns-kbx4w.svc.cluster.local:8265/api/jobs/delete-head-after-submit-cwfp8\": dial tcp 10.134.0.49:8265: connect: connection refused","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:321\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:30:43.858Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"06e29344-8d5e-4f2c-8913-c0e7b6842e1b","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:30:43.858Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"06e29344-8d5e-4f2c-8913-c0e7b6842e1b","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:30:43.858Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"06e29344-8d5e-4f2c-8913-c0e7b6842e1b","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:30:43.858Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"06e29344-8d5e-4f2c-8913-c0e7b6842e1b"}
{"level":"info","ts":"2026-11-03T15:30:43.858Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"06e29344-8d5e-4f2c-8913-c0e7b6842e1b","rayCluster":"delete-head-after-submit-gg2wh","serviceType":"headService"}
{"level":"error","ts":"2026-11-03T15:30:43.860Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"06e29344-8d5e-4f2c-8913-c0e7b6842e1b","JobId":"delete-head-after-submit-cwfp8","error":"Get \"http://delete-head-after-submit-gg2wh-head-svc.test-ns-kbx4w.svc.cluster.local:8265/api/jobs/delete-head-after-submit-cwfp8\": dial tcp 10.134.0.49:8265: connect: connection refused","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:321\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:30:46.861Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"4599e979-6a67-407f-94a9-fc998600b98c","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:30:46.861Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"4599e979-6a67-407f-94a9-fc998600b98c","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:30:46.861Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"4599e979-6a67-407f-94a9-fc998600b98c","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:30:46.862Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"4599e979-6a67-407f-94a9-fc998600b98c"}
{"level":"info","ts":"2026-11-03T15:30:46.862Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"4599e979-6a67-407f-94a9-fc998600b98c","rayCluster":"delete-head-after-submit-gg2wh","serviceType":"headService"}
{"level":"error","ts":"2026-11-03T15:30:46.864Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"4599e979-6a67-407f-94a9-fc998600b98c","JobId":"delete-head-after-submit-cwfp8","error":"Get \"http://delete-head-after-submit-gg2wh-head-svc.test-ns-kbx4w.svc.cluster.local:8265/api/jobs/delete-head-after-submit-cwfp8\": dial tcp 10.134.0.49:8265: connect: connection refused","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:321\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:30:48.970Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"57e7c0eb-b310-43c5-b442-217c7de49aff"}
{"level":"info","ts":"2026-11-03T15:30:48.970Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"57e7c0eb-b310-43c5-b442-217c7de49aff"}
{"level":"info","ts":"2026-11-03T15:30:48.970Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"57e7c0eb-b310-43c5-b442-217c7de49aff","1 head service found":"delete-head-after-submit-gg2wh-head-svc"}
{"level":"info","ts":"2026-11-03T15:30:48.970Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"57e7c0eb-b310-43c5-b442-217c7de49aff","Found 1 head Pod":"delete-head-after-submit-gg2wh-head-mscjq","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:30:48.970Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"57e7c0eb-b310-43c5-b442-217c7de49aff","head Pod":"delete-head-after-submit-gg2wh-head-mscjq","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod delete-head-after-submit-gg2wh-head-mscjq. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:30:48.970Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"57e7c0eb-b310-43c5-b442-217c7de49aff","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:30:48.970Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"57e7c0eb-b310-43c5-b442-217c7de49aff","worker Pod":"delete-head-after-submit-gg2wh-small-group-worker-dxpfb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod delete-head-after-submit-gg2wh-small-group-worker-dxpfb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:30:48.970Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"57e7c0eb-b310-43c5-b442-217c7de49aff","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:30:48.970Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"57e7c0eb-b310-43c5-b442-217c7de49aff","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:30:48.970Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"57e7c0eb-b310-43c5-b442-217c7de49aff","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:30:48.970Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"57e7c0eb-b310-43c5-b442-217c7de49aff","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:30:48.970Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"57e7c0eb-b310-43c5-b442-217c7de49aff","podName":"delete-head-after-submit-gg2wh-head-mscjq","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:30:48.970Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"57e7c0eb-b310-43c5-b442-217c7de49aff","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:30:48.970Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"57e7c0eb-b310-43c5-b442-217c7de49aff","seconds":300}
{"level":"info","ts":"2026-11-03T15:30:49.865Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"49b1e297-2766-46c8-8f24-94c341facb37","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:30:49.865Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"49b1e297-2766-46c8-8f24-94c341facb37","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:30:49.865Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"49b1e297-2766-46c8-8f24-94c341facb37","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:30:49.865Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"49b1e297-2766-46c8-8f24-94c341facb37"}
{"level":"info","ts":"2026-11-03T15:30:49.865Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"49b1e297-2766-46c8-8f24-94c341facb37","rayCluster":"delete-head-after-submit-gg2wh","serviceType":"headService"}
{"level":"error","ts":"2026-11-03T15:30:49.886Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"49b1e297-2766-46c8-8f24-94c341facb37","JobId":"delete-head-after-submit-cwfp8","error":"Get \"http://delete-head-after-submit-gg2wh-head-svc.test-ns-kbx4w.svc.cluster.local:8265/api/jobs/delete-head-after-submit-cwfp8\": dial tcp 10.134.0.49:8265: connect: connection refused","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:321\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:30:50.014Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"720160f8-f1f7-429e-b0be-30c226c5750f"}
{"level":"info","ts":"2026-11-03T15:30:50.014Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"720160f8-f1f7-429e-b0be-30c226c5750f"}
{"level":"info","ts":"2026-11-03T15:30:50.014Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"720160f8-f1f7-429e-b0be-30c226c5750f","1 head service found":"delete-head-after-submit-gg2wh-head-svc"}
{"level":"info","ts":"2026-11-03T15:30:50.014Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"720160f8-f1f7-429e-b0be-30c226c5750f","Found 1 head Pod":"delete-head-after-submit-gg2wh-head-mscjq","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:30:50.014Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"720160f8-f1f7-429e-b0be-30c226c5750f","head Pod":"delete-head-after-submit-gg2wh-head-mscjq","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod delete-head-after-submit-gg2wh-head-mscjq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:30:50.014Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"720160f8-f1f7-429e-b0be-30c226c5750f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:30:50.014Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"720160f8-f1f7-429e-b0be-30c226c5750f","worker Pod":"delete-head-after-submit-gg2wh-small-group-worker-dxpfb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod delete-head-after-submit-gg2wh-small-group-worker-dxpfb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:30:50.014Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"720160f8-f1f7-429e-b0be-30c226c5750f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:30:50.014Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"720160f8-f1f7-429e-b0be-30c226c5750f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:30:50.014Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"720160f8-f1f7-429e-b0be-30c226c5750f","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:30:50.014Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"720160f8-f1f7-429e-b0be-30c226c5750f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:30:50.015Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"720160f8-f1f7-429e-b0be-30c226c5750f","podName":"delete-head-after-submit-gg2wh-head-mscjq","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:29:37Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:30:50.015Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"720160f8-f1f7-429e-b0be-30c226c5750f","name":"delete-head-after-submit-gg2wh","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:30:37Z","stateTransitionTimes":{"ready":"2026-11-03T15:29:33Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.49","serviceIP":"10.134.0.49","podName":"delete-head-after-submit-gg2wh-head-mscjq","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:29:37Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:29:33Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:30:50Z","stateTransitionTimes":{"ready":"2026-11-03T15:29:33Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.49","serviceIP":"10.134.0.49","podName":"delete-head-after-submit-gg2wh-head-mscjq","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:29:37Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:29:33Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:30:50.077Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"90a1e07c-08ed-4b82-b2be-753696dd5384","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:30:50.077Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"90a1e07c-08ed-4b82-b2be-753696dd5384","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:30:50.077Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"90a1e07c-08ed-4b82-b2be-753696dd5384","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:30:50.077Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"90a1e07c-08ed-4b82-b2be-753696dd5384"}
{"level":"info","ts":"2026-11-03T15:30:50.077Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"e1093c45-fba7-47d4-9292-fc3c55140ac6","cluster":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:30:50.077Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"90a1e07c-08ed-4b82-b2be-753696dd5384","rayCluster":"delete-head-after-submit-gg2wh","serviceType":"headService"}
{"level":"error","ts":"2026-11-03T15:30:50.079Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"90a1e07c-08ed-4b82-b2be-753696dd5384","JobId":"delete-head-after-submit-cwfp8","error":"Get \"http://delete-head-after-submit-gg2wh-head-svc.test-ns-kbx4w.svc.cluster.local:8265/api/jobs/delete-head-after-submit-cwfp8\": dial tcp 10.134.0.49:8265: connect: connection refused","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:321\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:30:52.080Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"0c1c67da-f964-4e72-b485-628cc7621118"}
{"level":"info","ts":"2026-11-03T15:30:52.080Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"0c1c67da-f964-4e72-b485-628cc7621118"}
{"level":"info","ts":"2026-11-03T15:30:52.081Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"0c1c67da-f964-4e72-b485-628cc7621118","1 head service found":"delete-head-after-submit-gg2wh-head-svc"}
{"level":"info","ts":"2026-11-03T15:30:52.081Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"0c1c67da-f964-4e72-b485-628cc7621118","Found 1 head Pod":"delete-head-after-submit-gg2wh-head-mscjq","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:30:52.081Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"0c1c67da-f964-4e72-b485-628cc7621118","head Pod":"delete-head-after-submit-gg2wh-head-mscjq","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod delete-head-after-submit-gg2wh-head-mscjq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:30:52.081Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"0c1c67da-f964-4e72-b485-628cc7621118","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:30:52.081Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"0c1c67da-f964-4e72-b485-628cc7621118","worker Pod":"delete-head-after-submit-gg2wh-small-group-worker-dxpfb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod delete-head-after-submit-gg2wh-small-group-worker-dxpfb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:30:52.081Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"0c1c67da-f964-4e72-b485-628cc7621118","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:30:52.081Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"0c1c67da-f964-4e72-b485-628cc7621118","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:30:52.081Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"0c1c67da-f964-4e72-b485-628cc7621118","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:30:52.081Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"0c1c67da-f964-4e72-b485-628cc7621118","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:30:52.081Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"0c1c67da-f964-4e72-b485-628cc7621118","podName":"delete-head-after-submit-gg2wh-head-mscjq","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:29:37Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:30:52.081Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"0c1c67da-f964-4e72-b485-628cc7621118","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:30:52.081Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"0c1c67da-f964-4e72-b485-628cc7621118","seconds":300}
{"level":"info","ts":"2026-11-03T15:30:53.082Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"d6dd4e80-e5bb-4fbc-828f-15f52bb70180","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:30:53.082Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"d6dd4e80-e5bb-4fbc-828f-15f52bb70180","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:30:53.083Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"d6dd4e80-e5bb-4fbc-828f-15f52bb70180","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:30:53.083Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"d6dd4e80-e5bb-4fbc-828f-15f52bb70180"}
{"level":"info","ts":"2026-11-03T15:30:53.083Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"d6dd4e80-e5bb-4fbc-828f-15f52bb70180","rayCluster":"delete-head-after-submit-gg2wh","serviceType":"headService"}
{"level":"error","ts":"2026-11-03T15:30:53.085Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"d6dd4e80-e5bb-4fbc-828f-15f52bb70180","JobId":"delete-head-after-submit-cwfp8","error":"Get \"http://delete-head-after-submit-gg2wh-head-svc.test-ns-kbx4w.svc.cluster.local:8265/api/jobs/delete-head-after-submit-cwfp8\": dial tcp 10.134.0.49:8265: connect: connection refused","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:321\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:30:56.086Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"15a1547e-b616-4d64-b383-3e65fa3e4143","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:30:56.086Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"15a1547e-b616-4d64-b383-3e65fa3e4143","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:30:56.086Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"15a1547e-b616-4d64-b383-3e65fa3e4143","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:30:56.087Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"15a1547e-b616-4d64-b383-3e65fa3e4143"}
{"level":"info","ts":"2026-11-03T15:30:56.087Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"15a1547e-b616-4d64-b383-3e65fa3e4143","rayCluster":"delete-head-after-submit-gg2wh","serviceType":"headService"}
{"level":"error","ts":"2026-11-03T15:30:56.089Z","logger":"controllers.RayJob","msg":"Failed to get job info","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"15a1547e-b616-4d64-b383-3e65fa3e4143","JobId":"delete-head-after-submit-cwfp8","error":"Get \"http://delete-head-after-submit-gg2wh-head-svc.test-ns-kbx4w.svc.cluster.local:8265/api/jobs/delete-head-after-submit-cwfp8\": dial tcp 10.134.0.49:8265: connect: connection refused","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:321\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:30:59.090Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"79a2a67a-1bb0-4c75-a24e-80a58c7471e7","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:30:59.090Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"79a2a67a-1bb0-4c75-a24e-80a58c7471e7","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:30:59.090Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"79a2a67a-1bb0-4c75-a24e-80a58c7471e7","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:30:59.091Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"79a2a67a-1bb0-4c75-a24e-80a58c7471e7"}
{"level":"info","ts":"2026-11-03T15:30:59.091Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"79a2a67a-1bb0-4c75-a24e-80a58c7471e7","rayCluster":"delete-head-after-submit-gg2wh","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:31:01.038Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"5d1f70f0-065a-49ce-8228-788b61f6b362"}
{"level":"info","ts":"2026-11-03T15:31:01.038Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"5d1f70f0-065a-49ce-8228-788b61f6b362"}
{"level":"info","ts":"2026-11-03T15:31:01.038Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"5d1f70f0-065a-49ce-8228-788b61f6b362","1 head service found":"delete-head-after-submit-gg2wh-head-svc"}
{"level":"info","ts":"2026-11-03T15:31:01.038Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"5d1f70f0-065a-49ce-8228-788b61f6b362","Found 1 head Pod":"delete-head-after-submit-gg2wh-head-mscjq","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:31:01.038Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"5d1f70f0-065a-49ce-8228-788b61f6b362","head Pod":"delete-head-after-submit-gg2wh-head-mscjq","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod delete-head-after-submit-gg2wh-head-mscjq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:31:01.039Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"5d1f70f0-065a-49ce-8228-788b61f6b362","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:31:01.039Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"5d1f70f0-065a-49ce-8228-788b61f6b362","worker Pod":"delete-head-after-submit-gg2wh-small-group-worker-dxpfb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod delete-head-after-submit-gg2wh-small-group-worker-dxpfb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:31:01.039Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"5d1f70f0-065a-49ce-8228-788b61f6b362","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:31:01.039Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"5d1f70f0-065a-49ce-8228-788b61f6b362","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:31:01.039Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"5d1f70f0-065a-49ce-8228-788b61f6b362","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:31:01.039Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"5d1f70f0-065a-49ce-8228-788b61f6b362","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:31:01.039Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"5d1f70f0-065a-49ce-8228-788b61f6b362","podName":"delete-head-after-submit-gg2wh-small-group-worker-dxpfb","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:30:37Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:31:01.039Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"5d1f70f0-065a-49ce-8228-788b61f6b362","name":"delete-head-after-submit-gg2wh","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:30:50Z","stateTransitionTimes":{"ready":"2026-11-03T15:29:33Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.49","serviceIP":"10.134.0.49","podName":"delete-head-after-submit-gg2wh-head-mscjq","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:29:37Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:29:33Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:31:01Z","stateTransitionTimes":{"ready":"2026-11-03T15:29:33Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.49","serviceIP":"10.134.0.49","podName":"delete-head-after-submit-gg2wh-head-mscjq","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:31:01Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:29:33Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:31:01.102Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"bf4c8b86-4570-4147-a4dc-fd94e91dabb7","cluster":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:31:01.103Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"b44a0d17-8313-40da-8595-4afc42950713","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:31:01.103Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"b44a0d17-8313-40da-8595-4afc42950713","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:31:01.103Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"b44a0d17-8313-40da-8595-4afc42950713","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:31:01.103Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"b44a0d17-8313-40da-8595-4afc42950713"}
{"level":"info","ts":"2026-11-03T15:31:01.103Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"b44a0d17-8313-40da-8595-4afc42950713","rayCluster":"delete-head-after-submit-gg2wh","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:31:03.103Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"d0afe5fb-6f01-4a8f-9cc1-e5de1f7173bd"}
{"level":"info","ts":"2026-11-03T15:31:03.103Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"d0afe5fb-6f01-4a8f-9cc1-e5de1f7173bd"}
{"level":"info","ts":"2026-11-03T15:31:03.103Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"d0afe5fb-6f01-4a8f-9cc1-e5de1f7173bd","1 head service found":"delete-head-after-submit-gg2wh-head-svc"}
{"level":"info","ts":"2026-11-03T15:31:03.103Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"d0afe5fb-6f01-4a8f-9cc1-e5de1f7173bd","Found 1 head Pod":"delete-head-after-submit-gg2wh-head-mscjq","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:31:03.103Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"d0afe5fb-6f01-4a8f-9cc1-e5de1f7173bd","head Pod":"delete-head-after-submit-gg2wh-head-mscjq","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod delete-head-after-submit-gg2wh-head-mscjq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:31:03.103Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"d0afe5fb-6f01-4a8f-9cc1-e5de1f7173bd","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:31:03.103Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"d0afe5fb-6f01-4a8f-9cc1-e5de1f7173bd","worker Pod":"delete-head-after-submit-gg2wh-small-group-worker-dxpfb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod delete-head-after-submit-gg2wh-small-group-worker-dxpfb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:31:03.103Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"d0afe5fb-6f01-4a8f-9cc1-e5de1f7173bd","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:31:03.103Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"d0afe5fb-6f01-4a8f-9cc1-e5de1f7173bd","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:31:03.103Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"d0afe5fb-6f01-4a8f-9cc1-e5de1f7173bd","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:31:03.103Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"d0afe5fb-6f01-4a8f-9cc1-e5de1f7173bd","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:31:03.103Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"d0afe5fb-6f01-4a8f-9cc1-e5de1f7173bd","podName":"delete-head-after-submit-gg2wh-small-group-worker-dxpfb","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:30:37Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:31:03.103Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"d0afe5fb-6f01-4a8f-9cc1-e5de1f7173bd","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:31:03.103Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"d0afe5fb-6f01-4a8f-9cc1-e5de1f7173bd","seconds":300}
{"level":"info","ts":"2026-11-03T15:31:04.108Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"4a384b70-16d1-48bb-bc53-c3d3bb76dacc","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:31:04.109Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"4a384b70-16d1-48bb-bc53-c3d3bb76dacc","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:31:04.109Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"4a384b70-16d1-48bb-bc53-c3d3bb76dacc","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:31:04.109Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"4a384b70-16d1-48bb-bc53-c3d3bb76dacc"}
{"level":"info","ts":"2026-11-03T15:31:04.109Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"4a384b70-16d1-48bb-bc53-c3d3bb76dacc","rayCluster":"delete-head-after-submit-gg2wh","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:31:04.801Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"7e4b4e62-7ea3-4537-b88c-f5dd19ce9bb3"}
{"level":"info","ts":"2026-11-03T15:31:04.801Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"7e4b4e62-7ea3-4537-b88c-f5dd19ce9bb3"}
{"level":"info","ts":"2026-11-03T15:31:04.801Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"7e4b4e62-7ea3-4537-b88c-f5dd19ce9bb3","1 head service found":"delete-head-after-submit-gg2wh-head-svc"}
{"level":"info","ts":"2026-11-03T15:31:04.801Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"7e4b4e62-7ea3-4537-b88c-f5dd19ce9bb3","Found 1 head Pod":"delete-head-after-submit-gg2wh-head-mscjq","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:31:04.801Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"7e4b4e62-7ea3-4537-b88c-f5dd19ce9bb3","head Pod":"delete-head-after-submit-gg2wh-head-mscjq","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod delete-head-after-submit-gg2wh-head-mscjq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:31:04.801Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"7e4b4e62-7ea3-4537-b88c-f5dd19ce9bb3","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:31:04.801Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"7e4b4e62-7ea3-4537-b88c-f5dd19ce9bb3","worker Pod":"delete-head-after-submit-gg2wh-small-group-worker-dxpfb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod delete-head-after-submit-gg2wh-small-group-worker-dxpfb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:31:04.801Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"7e4b4e62-7ea3-4537-b88c-f5dd19ce9bb3","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:31:04.801Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"7e4b4e62-7ea3-4537-b88c-f5dd19ce9bb3","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:31:04.801Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"7e4b4e62-7ea3-4537-b88c-f5dd19ce9bb3","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:31:04.801Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"7e4b4e62-7ea3-4537-b88c-f5dd19ce9bb3","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:31:04.802Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"7e4b4e62-7ea3-4537-b88c-f5dd19ce9bb3","name":"delete-head-after-submit-gg2wh","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:31:01Z","stateTransitionTimes":{"ready":"2026-11-03T15:29:33Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.49","serviceIP":"10.134.0.49","podName":"delete-head-after-submit-gg2wh-head-mscjq","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:31:01Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:29:33Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:31:04Z","stateTransitionTimes":{"ready":"2026-11-03T15:29:33Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.49","serviceIP":"10.134.0.49","podName":"delete-head-after-submit-gg2wh-head-mscjq","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:31:01Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:29:33Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:31:04.863Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"068ef305-a27e-4aa0-9a18-b9e0a2ea0a70","cluster":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:31:04.863Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"18096825-60eb-47c9-99f4-46b2692f1551","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:31:04.863Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"18096825-60eb-47c9-99f4-46b2692f1551","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:31:04.863Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"18096825-60eb-47c9-99f4-46b2692f1551","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:31:04.863Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"18096825-60eb-47c9-99f4-46b2692f1551"}
{"level":"info","ts":"2026-11-03T15:31:04.864Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"18096825-60eb-47c9-99f4-46b2692f1551","rayCluster":"delete-head-after-submit-gg2wh","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:31:06.863Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"c130e8eb-9da2-4f08-bdf3-82867d7c5cca"}
{"level":"info","ts":"2026-11-03T15:31:06.863Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"c130e8eb-9da2-4f08-bdf3-82867d7c5cca"}
{"level":"info","ts":"2026-11-03T15:31:06.863Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"c130e8eb-9da2-4f08-bdf3-82867d7c5cca","1 head service found":"delete-head-after-submit-gg2wh-head-svc"}
{"level":"info","ts":"2026-11-03T15:31:06.863Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"c130e8eb-9da2-4f08-bdf3-82867d7c5cca","Found 1 head Pod":"delete-head-after-submit-gg2wh-head-mscjq","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:31:06.863Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"c130e8eb-9da2-4f08-bdf3-82867d7c5cca","head Pod":"delete-head-after-submit-gg2wh-head-mscjq","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod delete-head-after-submit-gg2wh-head-mscjq. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:31:06.863Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"c130e8eb-9da2-4f08-bdf3-82867d7c5cca","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:31:06.863Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"c130e8eb-9da2-4f08-bdf3-82867d7c5cca","worker Pod":"delete-head-after-submit-gg2wh-small-group-worker-dxpfb","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod delete-head-after-submit-gg2wh-small-group-worker-dxpfb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:31:06.863Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"c130e8eb-9da2-4f08-bdf3-82867d7c5cca","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:31:06.863Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"c130e8eb-9da2-4f08-bdf3-82867d7c5cca","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:31:06.863Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"c130e8eb-9da2-4f08-bdf3-82867d7c5cca","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:31:06.863Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"c130e8eb-9da2-4f08-bdf3-82867d7c5cca","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:31:06.864Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"c130e8eb-9da2-4f08-bdf3-82867d7c5cca","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:31:06.864Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"c130e8eb-9da2-4f08-bdf3-82867d7c5cca","seconds":300}
{"level":"info","ts":"2026-11-03T15:31:07.870Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"a90db0bd-df3c-4b6d-959c-1b487788733e","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:31:07.870Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"a90db0bd-df3c-4b6d-959c-1b487788733e","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:31:07.870Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"a90db0bd-df3c-4b6d-959c-1b487788733e","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:31:07.870Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"a90db0bd-df3c-4b6d-959c-1b487788733e"}
{"level":"info","ts":"2026-11-03T15:31:07.870Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"a90db0bd-df3c-4b6d-959c-1b487788733e","rayCluster":"delete-head-after-submit-gg2wh","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:31:10.877Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"c9c31f9c-0604-4f69-90e7-d52468d917c3","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:31:10.878Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"c9c31f9c-0604-4f69-90e7-d52468d917c3","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:31:10.878Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"c9c31f9c-0604-4f69-90e7-d52468d917c3","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:31:10.878Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"c9c31f9c-0604-4f69-90e7-d52468d917c3"}
{"level":"info","ts":"2026-11-03T15:31:10.878Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"c9c31f9c-0604-4f69-90e7-d52468d917c3","rayCluster":"delete-head-after-submit-gg2wh","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:31:13.884Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"db2f1a70-3266-4867-a7f1-fe97a7a5887d","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:31:13.884Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"db2f1a70-3266-4867-a7f1-fe97a7a5887d","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:31:13.884Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"db2f1a70-3266-4867-a7f1-fe97a7a5887d","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:31:13.884Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"db2f1a70-3266-4867-a7f1-fe97a7a5887d"}
{"level":"info","ts":"2026-11-03T15:31:13.884Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"db2f1a70-3266-4867-a7f1-fe97a7a5887d","rayCluster":"delete-head-after-submit-gg2wh","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:31:16.963Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"b3eadc54-0f81-4fd4-a8aa-84431e78d76e","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:31:16.963Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"b3eadc54-0f81-4fd4-a8aa-84431e78d76e","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:31:16.963Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"b3eadc54-0f81-4fd4-a8aa-84431e78d76e","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:31:16.963Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"b3eadc54-0f81-4fd4-a8aa-84431e78d76e"}
{"level":"info","ts":"2026-11-03T15:31:16.963Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"b3eadc54-0f81-4fd4-a8aa-84431e78d76e","rayCluster":"delete-head-after-submit-gg2wh","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:31:19.970Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"85c378f2-516c-4f90-8923-3ba6f45d725c","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:31:19.970Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"85c378f2-516c-4f90-8923-3ba6f45d725c","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:31:19.970Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"85c378f2-516c-4f90-8923-3ba6f45d725c","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:31:19.970Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"85c378f2-516c-4f90-8923-3ba6f45d725c"}
{"level":"info","ts":"2026-11-03T15:31:19.970Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"85c378f2-516c-4f90-8923-3ba6f45d725c","rayCluster":"delete-head-after-submit-gg2wh","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:31:22.976Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"738fce0e-c1fa-41a2-85c8-fd3937f9aa0a","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:31:22.976Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"738fce0e-c1fa-41a2-85c8-fd3937f9aa0a","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:31:22.976Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"738fce0e-c1fa-41a2-85c8-fd3937f9aa0a","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:31:22.977Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"738fce0e-c1fa-41a2-85c8-fd3937f9aa0a"}
{"level":"info","ts":"2026-11-03T15:31:22.977Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"738fce0e-c1fa-41a2-85c8-fd3937f9aa0a","rayCluster":"delete-head-after-submit-gg2wh","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:31:25.983Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"44cee3a4-fd8e-4212-9df9-0afe812580e8","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:31:25.983Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"44cee3a4-fd8e-4212-9df9-0afe812580e8","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:31:25.983Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"44cee3a4-fd8e-4212-9df9-0afe812580e8","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:31:25.983Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"44cee3a4-fd8e-4212-9df9-0afe812580e8"}
{"level":"info","ts":"2026-11-03T15:31:25.983Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"44cee3a4-fd8e-4212-9df9-0afe812580e8","rayCluster":"delete-head-after-submit-gg2wh","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:31:28.988Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"f76f1987-698b-4937-9475-8ad595ad298d","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:31:28.988Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"f76f1987-698b-4937-9475-8ad595ad298d","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:31:28.988Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"f76f1987-698b-4937-9475-8ad595ad298d","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:31:28.988Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"f76f1987-698b-4937-9475-8ad595ad298d"}
{"level":"info","ts":"2026-11-03T15:31:28.989Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"f76f1987-698b-4937-9475-8ad595ad298d","rayCluster":"delete-head-after-submit-gg2wh","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:31:31.995Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"a8d43356-112d-4b6a-a033-e52c41eee6d5","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:31:31.995Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"a8d43356-112d-4b6a-a033-e52c41eee6d5","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:31:31.995Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"a8d43356-112d-4b6a-a033-e52c41eee6d5","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:31:31.995Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"a8d43356-112d-4b6a-a033-e52c41eee6d5"}
{"level":"info","ts":"2026-11-03T15:31:31.995Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"a8d43356-112d-4b6a-a033-e52c41eee6d5","rayCluster":"delete-head-after-submit-gg2wh","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:31:35.001Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"8e0da75f-8820-4cb7-b66b-5f56da0928cf","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:31:35.001Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"8e0da75f-8820-4cb7-b66b-5f56da0928cf","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:31:35.001Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"8e0da75f-8820-4cb7-b66b-5f56da0928cf","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:31:35.002Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"8e0da75f-8820-4cb7-b66b-5f56da0928cf"}
{"level":"info","ts":"2026-11-03T15:31:35.002Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"8e0da75f-8820-4cb7-b66b-5f56da0928cf","rayCluster":"delete-head-after-submit-gg2wh","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:31:38.007Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"81fbc1e2-f84f-4456-9783-969438eb6f48","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:31:38.007Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"81fbc1e2-f84f-4456-9783-969438eb6f48","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:31:38.008Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"81fbc1e2-f84f-4456-9783-969438eb6f48","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:31:38.008Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"81fbc1e2-f84f-4456-9783-969438eb6f48"}
{"level":"info","ts":"2026-11-03T15:31:38.008Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"81fbc1e2-f84f-4456-9783-969438eb6f48","rayCluster":"delete-head-after-submit-gg2wh","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:31:39.790Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"ef44d57a-cf27-4e72-bb00-21cf088a677a","cluster":"long-running-hvml9"}
{"level":"info","ts":"2026-11-03T15:31:41.017Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"85208995-9249-46f0-9334-101903b47af2","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:31:41.017Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"85208995-9249-46f0-9334-101903b47af2","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:31:41.017Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"85208995-9249-46f0-9334-101903b47af2","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:31:41.017Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"85208995-9249-46f0-9334-101903b47af2"}
{"level":"info","ts":"2026-11-03T15:31:41.017Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"85208995-9249-46f0-9334-101903b47af2","rayCluster":"delete-head-after-submit-gg2wh","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:31:44.024Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"03200259-8a38-4832-a581-4ea12b000588","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:31:44.024Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"03200259-8a38-4832-a581-4ea12b000588","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:31:44.024Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"03200259-8a38-4832-a581-4ea12b000588","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:31:44.024Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"03200259-8a38-4832-a581-4ea12b000588"}
{"level":"info","ts":"2026-11-03T15:31:44.024Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"03200259-8a38-4832-a581-4ea12b000588","rayCluster":"delete-head-after-submit-gg2wh","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:31:47.029Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"87fbdc4a-72d4-4e73-b3be-79e389a630eb","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:31:47.029Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"87fbdc4a-72d4-4e73-b3be-79e389a630eb","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:31:47.030Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"87fbdc4a-72d4-4e73-b3be-79e389a630eb","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:31:47.030Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"87fbdc4a-72d4-4e73-b3be-79e389a630eb"}
{"level":"info","ts":"2026-11-03T15:31:47.030Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"87fbdc4a-72d4-4e73-b3be-79e389a630eb","rayCluster":"delete-head-after-submit-gg2wh","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:31:50.035Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"5848b012-56a2-431e-bbe3-c0de85cdd8b2","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:31:50.036Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"5848b012-56a2-431e-bbe3-c0de85cdd8b2","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:31:50.036Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"5848b012-56a2-431e-bbe3-c0de85cdd8b2","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:31:50.036Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"5848b012-56a2-431e-bbe3-c0de85cdd8b2"}
{"level":"info","ts":"2026-11-03T15:31:50.036Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"5848b012-56a2-431e-bbe3-c0de85cdd8b2","rayCluster":"delete-head-after-submit-gg2wh","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:31:52.000Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"d86928c0-3dec-49dd-ae64-639258ca17f9","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:31:52.000Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"d86928c0-3dec-49dd-ae64-639258ca17f9","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:31:52.000Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"d86928c0-3dec-49dd-ae64-639258ca17f9","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:31:52.000Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"d86928c0-3dec-49dd-ae64-639258ca17f9"}
{"level":"info","ts":"2026-11-03T15:31:52.000Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"d86928c0-3dec-49dd-ae64-639258ca17f9","rayCluster":"delete-head-after-submit-gg2wh","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:31:53.009Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"1cc3115d-71ad-4f09-9203-b712f51fc2a9","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:31:53.009Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"1cc3115d-71ad-4f09-9203-b712f51fc2a9","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:31:53.009Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"1cc3115d-71ad-4f09-9203-b712f51fc2a9","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:31:53.009Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"1cc3115d-71ad-4f09-9203-b712f51fc2a9"}
{"level":"info","ts":"2026-11-03T15:31:53.009Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"1cc3115d-71ad-4f09-9203-b712f51fc2a9","rayCluster":"delete-head-after-submit-gg2wh","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:31:53.029Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"33b9cdcc-93a0-49e5-bb50-50bb10cb6974","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:31:53.029Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"33b9cdcc-93a0-49e5-bb50-50bb10cb6974","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:31:53.029Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"33b9cdcc-93a0-49e5-bb50-50bb10cb6974","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:31:53.029Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"33b9cdcc-93a0-49e5-bb50-50bb10cb6974"}
{"level":"info","ts":"2026-11-03T15:31:53.029Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"33b9cdcc-93a0-49e5-bb50-50bb10cb6974","rayCluster":"delete-head-after-submit-gg2wh","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:31:53.029Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job has failed. Attempting to transition the status to `Failed`.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"33b9cdcc-93a0-49e5-bb50-50bb10cb6974","Submitter K8s Job":"delete-head-after-submit","Reason":"BackoffLimitExceeded","Message":"Job has reached the specified backoff limit"}
{"level":"info","ts":"2026-11-03T15:31:53.029Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"33b9cdcc-93a0-49e5-bb50-50bb10cb6974","oldRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","dashboardURL":"delete-head-after-submit-gg2wh-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:29:22Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:29:01Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"jobStatusCheckFailureStartTime":"2026-11-03T15:29:36Z"},"newRayJobStatus":{"rayJobInfo":{},"jobId":"delete-head-after-submit-cwfp8","rayClusterName":"delete-head-after-submit-gg2wh","dashboardURL":"delete-head-after-submit-gg2wh-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobDeploymentStatus":"Failed","reason":"SubmissionFailed","message":"Job submission has failed. Reason: BackoffLimitExceeded. Message: Job has reached the specified backoff limit","startTime":"2026-11-03T15:27:39Z","succeeded":0,"failed":1,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:29:22Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.48","serviceIP":"10.134.0.48","podName":"delete-head-after-submit-gg2wh-head-b99bx","serviceName":"delete-head-after-submit-gg2wh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:29:01Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:27:39Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:27:40Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"jobStatusCheckFailureStartTime":"2026-11-03T15:29:36Z"}}
{"level":"info","ts":"2026-11-03T15:31:53.029Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"33b9cdcc-93a0-49e5-bb50-50bb10cb6974","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Failed"}
{"level":"info","ts":"2026-11-03T15:31:53.091Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"3d765a44-f0e7-4c7a-b439-985ad0e60dcd","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:31:53.091Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"3d765a44-f0e7-4c7a-b439-985ad0e60dcd","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Failed","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-11-03 15:31:53 +0000 UTC","Now":"2026-11-03T15:31:53.091Z","ShutdownTime":"2026-11-03T15:31:53.000Z"}
{"level":"info","ts":"2026-11-03T15:31:53.150Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob is deleted","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"3d765a44-f0e7-4c7a-b439-985ad0e60dcd","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:31:53.150Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"3d765a44-f0e7-4c7a-b439-985ad0e60dcd","isClusterDeleted":false}
{"level":"info","ts":"2026-11-03T15:31:53.150Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"3d765a44-f0e7-4c7a-b439-985ad0e60dcd","deletionMechanism":"ShutdownAfterJobFinishes","RayCluster":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:31:53.150Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"reconcileID":"6e3c07ba-2973-4f2b-b248-9de7a546d833"}
{"level":"info","ts":"2026-11-03T15:31:53.151Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"delete-head-after-submit-gg2wh","reconcileID":"ce4f2f78-1432-4fed-b46e-f4a6bc4f871a","namespacedName":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:31:53.151Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"delete-head-after-submit-gg2wh","reconcileID":"ce4f2f78-1432-4fed-b46e-f4a6bc4f871a","cluster":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:31:53.151Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"delete-head-after-submit-gg2wh","reconcileID":"ce4f2f78-1432-4fed-b46e-f4a6bc4f871a","cluster":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:31:53.151Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"de1c3f2f-a6bb-4c60-88b9-6dc26a9dd8cc","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:31:53.151Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"de1c3f2f-a6bb-4c60-88b9-6dc26a9dd8cc","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Failed","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-11-03 15:31:53 +0000 UTC","Now":"2026-11-03T15:31:53.151Z","ShutdownTime":"2026-11-03T15:31:53.000Z"}
{"level":"info","ts":"2026-11-03T15:31:53.151Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"delete-head-after-submit-gg2wh","reconcileID":"ce4f2f78-1432-4fed-b46e-f4a6bc4f871a","namespace":"test-ns-kbx4w","otherAuthClusters":1,"action":"will retain"}
{"level":"info","ts":"2026-11-03T15:31:53.151Z","logger":"authentication-controller","msg":"ReferenceGrant retained (other clusters with auth exist)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"delete-head-after-submit-gg2wh","reconcileID":"ce4f2f78-1432-4fed-b46e-f4a6bc4f871a","name":"kuberay-gateway-access","namespace":"test-ns-kbx4w","remainingClusters":1}
{"level":"info","ts":"2026-11-03T15:31:53.151Z","logger":"controllers.RayJob","msg":"The deletion of the associated RayCluster for RayJob is ongoing.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"de1c3f2f-a6bb-4c60-88b9-6dc26a9dd8cc","RayCluster":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:31:53.151Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"de1c3f2f-a6bb-4c60-88b9-6dc26a9dd8cc","isClusterDeleted":false}
{"level":"info","ts":"2026-11-03T15:31:53.151Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"de1c3f2f-a6bb-4c60-88b9-6dc26a9dd8cc","deletionMechanism":"ShutdownAfterJobFinishes","RayCluster":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:31:53.204Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"delete-head-after-submit-gg2wh","reconcileID":"ce4f2f78-1432-4fed-b46e-f4a6bc4f871a","name":"test-ns-kbx4w-delete-head-after-submit-gg2wh","namespace":"opendatahub"}
{"level":"info","ts":"2026-11-03T15:31:53.259Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"delete-head-after-submit-gg2wh","reconcileID":"ce4f2f78-1432-4fed-b46e-f4a6bc4f871a","configMap":"kube-rbac-proxy-config-delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:31:53.311Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"delete-head-after-submit-gg2wh","reconcileID":"ce4f2f78-1432-4fed-b46e-f4a6bc4f871a","serviceAccount":"delete-head-after-submit-gg2wh-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-11-03T15:31:53.387Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"38cce852-b7f3-480e-8062-21583d429b61","DeletionTimestamp":"2026-11-03 15:31:53 +0000 UTC"}
{"level":"error","ts":"2026-11-03T15:31:53.392Z","logger":"controllers.RayJob","msg":"Failed to stop job for RayJob","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"38cce852-b7f3-480e-8062-21583d429b61","error":"invalid character 'J' looking for beginning of value","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:128\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:31:53.392Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"38cce852-b7f3-480e-8062-21583d429b61","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-11-03T15:31:53.456Z","logger":"controllers.RayJob","msg":"Skipping RayJob managed by a custom controller","RayJob":{"name":"managed-externally","namespace":"test-ns-kbx4w"},"reconcileID":"f2d77b88-b68c-4503-83f9-832205bd2206","managed-by":"kueue.x-k8s.io/multikueue"}
{"level":"info","ts":"2026-11-03T15:31:53.456Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"22e8146c-26e6-4032-bcef-23b528a03d34"}
{"level":"info","ts":"2026-11-03T15:31:53.462Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"07cfdcaa-7ba7-48c5-8cd1-99cf2e217569"}
{"level":"info","ts":"2026-11-03T15:31:53.463Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"30c77d54-53c5-4d15-a392-b9b6e5bdfb7b"}
{"level":"info","ts":"2026-11-03T15:31:53.490Z","logger":"raycluster-resource","msg":"default","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:31:53.490Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:31:53.522Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"managed-externally","namespace":"test-ns-kbx4w"},"reconcileID":"a9d2810d-f0d8-4696-aa84-8bb2c40b14d4"}
{"level":"info","ts":"2026-11-03T15:31:53.549Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"a234d42e-30da-441b-b401-6a4a6679ad2b","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:31:53.549Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"a234d42e-30da-441b-b401-6a4a6679ad2b","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-11-03T15:31:53.610Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"a234d42e-30da-441b-b401-6a4a6679ad2b"}
{"level":"info","ts":"2026-11-03T15:31:53.610Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"a234d42e-30da-441b-b401-6a4a6679ad2b","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-11-03T15:31:53.610Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"a234d42e-30da-441b-b401-6a4a6679ad2b","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:31:53.610Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"a234d42e-30da-441b-b401-6a4a6679ad2b","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:31:53.670Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"2375df53-2e02-42dd-ab51-ab8129a5dc19","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:31:53.670Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"2375df53-2e02-42dd-ab51-ab8129a5dc19","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:31:53.670Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"2375df53-2e02-42dd-ab51-ab8129a5dc19","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:31:53.670Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"2375df53-2e02-42dd-ab51-ab8129a5dc19","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:31:53.671Z","logger":"raycluster-resource","msg":"validate update","name":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:31:53.724Z","logger":"raycluster-resource","msg":"default","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:31:53.724Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:31:53.726Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"delete-head-after-submit-gg2wh","reconcileID":"ce4f2f78-1432-4fed-b46e-f4a6bc4f871a","cluster":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:31:53.726Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"delete-head-after-submit-gg2wh","reconcileID":"356c80c0-9a5b-41bb-b94d-45ac6646ee96","namespacedName":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:31:53.726Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"delete-head-after-submit-gg2wh","reconcileID":"356c80c0-9a5b-41bb-b94d-45ac6646ee96","cluster":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:31:53.726Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"delete-head-after-submit-gg2wh","reconcileID":"356c80c0-9a5b-41bb-b94d-45ac6646ee96","cluster":"delete-head-after-submit-gg2wh"}
{"level":"info","ts":"2026-11-03T15:31:53.726Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"delete-head-after-submit-gg2wh","reconcileID":"356c80c0-9a5b-41bb-b94d-45ac6646ee96","namespace":"test-ns-kbx4w","otherAuthClusters":1,"action":"will retain"}
{"level":"info","ts":"2026-11-03T15:31:53.726Z","logger":"authentication-controller","msg":"ReferenceGrant retained (other clusters with auth exist)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"delete-head-after-submit-gg2wh","reconcileID":"356c80c0-9a5b-41bb-b94d-45ac6646ee96","name":"kuberay-gateway-access","namespace":"test-ns-kbx4w","remainingClusters":1}
{"level":"error","ts":"2026-11-03T15:31:53.777Z","logger":"authentication-controller","msg":"Failed to remove finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"delete-head-after-submit-gg2wh","reconcileID":"356c80c0-9a5b-41bb-b94d-45ac6646ee96","error":"rayclusters.ray.io \"delete-head-after-submit-gg2wh\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleDeletion\n\t/workspace/controllers/ray/authentication_controller.go:195\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:31:53.777Z","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"delete-head-after-submit-gg2wh","reconcileID":"356c80c0-9a5b-41bb-b94d-45ac6646ee96"}
{"level":"error","ts":"2026-11-03T15:31:53.777Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"delete-head-after-submit-gg2wh","reconcileID":"356c80c0-9a5b-41bb-b94d-45ac6646ee96","error":"rayclusters.ray.io \"delete-head-after-submit-gg2wh\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:31:53.778Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"delete-head-after-submit-gg2wh","reconcileID":"74dc5754-fd58-4400-8900-73ac55c89d16","namespacedName":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:31:53.778Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"delete-head-after-submit-gg2wh","reconcileID":"74dc5754-fd58-4400-8900-73ac55c89d16"}
{"level":"info","ts":"2026-11-03T15:31:53.778Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"delete-head-after-submit-gg2wh","reconcileID":"74dc5754-fd58-4400-8900-73ac55c89d16","namespace":"test-ns-kbx4w","authClusters":1}
{"level":"info","ts":"2026-11-03T15:31:53.778Z","logger":"authentication-controller","msg":"ReferenceGrant not orphaned (clusters with auth exist)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"delete-head-after-submit-gg2wh","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"delete-head-after-submit-gg2wh","reconcileID":"74dc5754-fd58-4400-8900-73ac55c89d16","namespace":"test-ns-kbx4w","authClusters":1}
{"level":"info","ts":"2026-11-03T15:31:53.789Z","logger":"raycluster-resource","msg":"validate create","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:31:53.843Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"2375df53-2e02-42dd-ab51-ab8129a5dc19","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:31:53.844Z","logger":"controllers.NetworkPolicy","msg":"Reconciling NetworkPolicies for RayCluster","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"37c65b10-3d23-4c55-a801-fecc212367c9","cluster":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:31:53.844Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"2375df53-2e02-42dd-ab51-ab8129a5dc19"}
{"level":"info","ts":"2026-11-03T15:31:53.844Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"2375df53-2e02-42dd-ab51-ab8129a5dc19","RayCluster":"submitter-timeout-g4hph","State":""}
{"level":"info","ts":"2026-11-03T15:31:53.844Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"2375df53-2e02-42dd-ab51-ab8129a5dc19","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:31:53.844Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"d7988322-522a-4abb-b12e-6d3790063993","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:31:53.844Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"d7988322-522a-4abb-b12e-6d3790063993","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:31:53.844Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"d7988322-522a-4abb-b12e-6d3790063993","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:31:53.844Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"d7988322-522a-4abb-b12e-6d3790063993"}
{"level":"info","ts":"2026-11-03T15:31:53.844Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"d7988322-522a-4abb-b12e-6d3790063993","RayCluster":"submitter-timeout-g4hph","State":""}
{"level":"info","ts":"2026-11-03T15:31:53.844Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"d7988322-522a-4abb-b12e-6d3790063993","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:31:53.844Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"66106079-d6a2-4bd6-aff2-8ead8a00c9ad"}
{"level":"info","ts":"2026-11-03T15:31:53.844Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"66106079-d6a2-4bd6-aff2-8ead8a00c9ad"}
{"level":"info","ts":"2026-11-03T15:31:53.844Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"delete-head-after-submit","namespace":"test-ns-kbx4w"},"reconcileID":"9e6730a9-7270-4260-949b-0689fe74d550"}
{"level":"info","ts":"2026-11-03T15:31:53.844Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"7b6a5a40-ed39-42af-acd5-c8ec313ebe2f","cluster":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:31:53.844Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"submitter-timeout-g4hph","reconcileID":"4bdb5c18-7ef7-49fd-a1f3-ce44254694d9","namespacedName":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:31:53.844Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"submitter-timeout-g4hph","reconcileID":"4bdb5c18-7ef7-49fd-a1f3-ce44254694d9","mode":"IntegratedOAuth","cluster":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:31:53.844Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"ab13096c-5005-45af-9230-d1e3cb348e63","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:31:53.844Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"ab13096c-5005-45af-9230-d1e3cb348e63","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:31:53.844Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"ab13096c-5005-45af-9230-d1e3cb348e63","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:31:53.844Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"submitter-timeout-g4hph","reconcileID":"4bdb5c18-7ef7-49fd-a1f3-ce44254694d9","cluster":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:31:53.844Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"submitter-timeout-g4hph","reconcileID":"4bdb5c18-7ef7-49fd-a1f3-ce44254694d9","cluster":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:31:53.844Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"ab13096c-5005-45af-9230-d1e3cb348e63"}
{"level":"info","ts":"2026-11-03T15:31:53.844Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"ab13096c-5005-45af-9230-d1e3cb348e63","RayCluster":"submitter-timeout-g4hph","State":""}
{"level":"info","ts":"2026-11-03T15:31:53.844Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"ab13096c-5005-45af-9230-d1e3cb348e63","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:31:53.899Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"66106079-d6a2-4bd6-aff2-8ead8a00c9ad","name":"submitter-timeout-g4hph-head-svc"}
{"level":"info","ts":"2026-11-03T15:31:53.899Z","logger":"controllers.NetworkPolicy","msg":"Successfully created NetworkPolicy","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"37c65b10-3d23-4c55-a801-fecc212367c9","name":"submitter-timeout-g4hph-head"}
{"level":"info","ts":"2026-11-03T15:31:53.899Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"66106079-d6a2-4bd6-aff2-8ead8a00c9ad"}
{"level":"info","ts":"2026-11-03T15:31:53.900Z","logger":"raycluster-resource","msg":"default","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:31:53.901Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:31:53.949Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"66106079-d6a2-4bd6-aff2-8ead8a00c9ad","error":"certificate ray-head-cert-submitter-timeout-g4hph not found: certificates.cert-manager.io \"ray-head-cert-submitter-timeout-g4hph\" not found"}
{"level":"error","ts":"2026-11-03T15:31:53.949Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"66106079-d6a2-4bd6-aff2-8ead8a00c9ad","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-submitter-timeout-g4hph not found: certificates.cert-manager.io \"ray-head-cert-submitter-timeout-g4hph\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:31:53.949Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"66106079-d6a2-4bd6-aff2-8ead8a00c9ad","error":"mTLS secrets not ready: certificate ray-head-cert-submitter-timeout-g4hph not found: certificates.cert-manager.io \"ray-head-cert-submitter-timeout-g4hph\" not found"}
{"level":"info","ts":"2026-11-03T15:31:53.949Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"66106079-d6a2-4bd6-aff2-8ead8a00c9ad","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"submitter-timeout-g4hph","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:31:53.949Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"66106079-d6a2-4bd6-aff2-8ead8a00c9ad","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"submitter-timeout-g4hph","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:31:53.949Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"66106079-d6a2-4bd6-aff2-8ead8a00c9ad","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"submitter-timeout-g4hph","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:31:53.949Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"66106079-d6a2-4bd6-aff2-8ead8a00c9ad","name":"submitter-timeout-g4hph","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:31:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:31:53.957Z","logger":"controllers.NetworkPolicy","msg":"Successfully created NetworkPolicy","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"37c65b10-3d23-4c55-a801-fecc212367c9","name":"submitter-timeout-g4hph-workers-small-group"}
{"level":"info","ts":"2026-11-03T15:31:53.957Z","logger":"controllers.NetworkPolicy","msg":"Successfully reconciled NetworkPolicies for RayCluster","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"37c65b10-3d23-4c55-a801-fecc212367c9","cluster":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:31:53.957Z","logger":"controllers.NetworkPolicy","msg":"Reconciling NetworkPolicies for RayCluster","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"fff725ee-bd76-473f-8149-5312b0d3c9bc","cluster":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:31:53.965Z","logger":"raycluster-resource","msg":"validate update","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:31:54.009Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"66106079-d6a2-4bd6-aff2-8ead8a00c9ad"}
{"level":"error","ts":"2026-11-03T15:31:54.009Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"66106079-d6a2-4bd6-aff2-8ead8a00c9ad","error":"mTLS secrets not ready: certificate ray-head-cert-submitter-timeout-g4hph not found: certificates.cert-manager.io \"ray-head-cert-submitter-timeout-g4hph\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:31:54.009Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"06187ff0-782e-4a48-b0f2-afa105cde935"}
{"level":"info","ts":"2026-11-03T15:31:54.009Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"06187ff0-782e-4a48-b0f2-afa105cde935"}
{"level":"info","ts":"2026-11-03T15:31:54.009Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"06187ff0-782e-4a48-b0f2-afa105cde935","1 head service found":"submitter-timeout-g4hph-head-svc"}
{"level":"info","ts":"2026-11-03T15:31:54.009Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"06187ff0-782e-4a48-b0f2-afa105cde935"}
{"level":"info","ts":"2026-11-03T15:31:54.009Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"5f26d932-3f93-483a-beaa-4c445be93931","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:31:54.009Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"5f26d932-3f93-483a-beaa-4c445be93931","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:31:54.009Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"5f26d932-3f93-483a-beaa-4c445be93931","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:31:54.010Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"5f26d932-3f93-483a-beaa-4c445be93931"}
{"level":"info","ts":"2026-11-03T15:31:54.010Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"5f26d932-3f93-483a-beaa-4c445be93931","RayCluster":"submitter-timeout-g4hph","State":""}
{"level":"info","ts":"2026-11-03T15:31:54.010Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"5f26d932-3f93-483a-beaa-4c445be93931","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:31:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:31:54.010Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"5f26d932-3f93-483a-beaa-4c445be93931","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:31:54.026Z","logger":"raycluster-resource","msg":"default","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:31:54.026Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:31:54.059Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"06187ff0-782e-4a48-b0f2-afa105cde935","error":"certificate ray-head-cert-submitter-timeout-g4hph not found: certificates.cert-manager.io \"ray-head-cert-submitter-timeout-g4hph\" not found"}
{"level":"error","ts":"2026-11-03T15:31:54.059Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"06187ff0-782e-4a48-b0f2-afa105cde935","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-submitter-timeout-g4hph not found: certificates.cert-manager.io \"ray-head-cert-submitter-timeout-g4hph\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:31:54.059Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"06187ff0-782e-4a48-b0f2-afa105cde935","error":"mTLS secrets not ready: certificate ray-head-cert-submitter-timeout-g4hph not found: certificates.cert-manager.io \"ray-head-cert-submitter-timeout-g4hph\" not found"}
{"level":"info","ts":"2026-11-03T15:31:54.059Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"06187ff0-782e-4a48-b0f2-afa105cde935","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"submitter-timeout-g4hph","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:31:54.059Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"06187ff0-782e-4a48-b0f2-afa105cde935","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"submitter-timeout-g4hph","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:31:54.059Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"06187ff0-782e-4a48-b0f2-afa105cde935","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"submitter-timeout-g4hph","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:31:54.059Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"06187ff0-782e-4a48-b0f2-afa105cde935","name":"submitter-timeout-g4hph","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:31:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:54Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:54Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:54Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:54Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:31:54.069Z","logger":"controllers.NetworkPolicy","msg":"Successfully reconciled NetworkPolicies for RayCluster","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"fff725ee-bd76-473f-8149-5312b0d3c9bc","cluster":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:31:54.071Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"adc172f7-b0e2-4c53-97cc-c6018cb1cb60","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:31:54.071Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"adc172f7-b0e2-4c53-97cc-c6018cb1cb60","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:31:54.071Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"adc172f7-b0e2-4c53-97cc-c6018cb1cb60","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:31:54.072Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"adc172f7-b0e2-4c53-97cc-c6018cb1cb60"}
{"level":"info","ts":"2026-11-03T15:31:54.072Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"adc172f7-b0e2-4c53-97cc-c6018cb1cb60","RayCluster":"submitter-timeout-g4hph","State":""}
{"level":"info","ts":"2026-11-03T15:31:54.072Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"adc172f7-b0e2-4c53-97cc-c6018cb1cb60","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:31:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:31:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"error","ts":"2026-11-03T15:31:54.075Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"submitter-timeout-g4hph","reconcileID":"4bdb5c18-7ef7-49fd-a1f3-ce44254694d9","error":"failed to add finalizer: Operation cannot be fulfilled on rayclusters.ray.io \"submitter-timeout-g4hph\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:167\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:31:54.075Z","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"submitter-timeout-g4hph","reconcileID":"4bdb5c18-7ef7-49fd-a1f3-ce44254694d9"}
{"level":"error","ts":"2026-11-03T15:31:54.075Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"submitter-timeout-g4hph","reconcileID":"4bdb5c18-7ef7-49fd-a1f3-ce44254694d9","error":"failed to add finalizer: Operation cannot be fulfilled on rayclusters.ray.io \"submitter-timeout-g4hph\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:31:54.075Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"submitter-timeout-g4hph","reconcileID":"6add30f2-6d28-411f-a53a-56ab5b71bfa9","namespacedName":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:31:54.075Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"submitter-timeout-g4hph","reconcileID":"6add30f2-6d28-411f-a53a-56ab5b71bfa9","mode":"IntegratedOAuth","cluster":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:31:54.075Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"submitter-timeout-g4hph","reconcileID":"6add30f2-6d28-411f-a53a-56ab5b71bfa9","cluster":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:31:54.075Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"submitter-timeout-g4hph","reconcileID":"6add30f2-6d28-411f-a53a-56ab5b71bfa9","cluster":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:31:54.114Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"06187ff0-782e-4a48-b0f2-afa105cde935","name":"submitter-timeout-g4hph","error":"Operation cannot be fulfilled on rayclusters.ray.io \"submitter-timeout-g4hph\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-kbx4w","name":"submitter-timeout-g4hph"}}
{"level":"info","ts":"2026-11-03T15:31:54.114Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"06187ff0-782e-4a48-b0f2-afa105cde935"}
{"level":"error","ts":"2026-11-03T15:31:54.114Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"06187ff0-782e-4a48-b0f2-afa105cde935","error":"mTLS secrets not ready: certificate ray-head-cert-submitter-timeout-g4hph not found: certificates.cert-manager.io \"ray-head-cert-submitter-timeout-g4hph\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:31:54.125Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"fe1339b8-daef-493a-af46-380e3f3d6574"}
{"level":"info","ts":"2026-11-03T15:31:54.125Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"fe1339b8-daef-493a-af46-380e3f3d6574"}
{"level":"info","ts":"2026-11-03T15:31:54.125Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"fe1339b8-daef-493a-af46-380e3f3d6574","1 head service found":"submitter-timeout-g4hph-head-svc"}
{"level":"info","ts":"2026-11-03T15:31:54.125Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"fe1339b8-daef-493a-af46-380e3f3d6574"}
{"level":"info","ts":"2026-11-03T15:31:54.132Z","logger":"raycluster-resource","msg":"default","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:31:54.132Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:31:54.174Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"fe1339b8-daef-493a-af46-380e3f3d6574","error":"certificate ray-head-cert-submitter-timeout-g4hph not found: certificates.cert-manager.io \"ray-head-cert-submitter-timeout-g4hph\" not found"}
{"level":"error","ts":"2026-11-03T15:31:54.175Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"fe1339b8-daef-493a-af46-380e3f3d6574","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-submitter-timeout-g4hph not found: certificates.cert-manager.io \"ray-head-cert-submitter-timeout-g4hph\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:31:54.175Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"fe1339b8-daef-493a-af46-380e3f3d6574","error":"mTLS secrets not ready: certificate ray-head-cert-submitter-timeout-g4hph not found: certificates.cert-manager.io \"ray-head-cert-submitter-timeout-g4hph\" not found"}
{"level":"info","ts":"2026-11-03T15:31:54.175Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"fe1339b8-daef-493a-af46-380e3f3d6574","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"submitter-timeout-g4hph","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:31:54.175Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"fe1339b8-daef-493a-af46-380e3f3d6574","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"submitter-timeout-g4hph","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:31:54.175Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"fe1339b8-daef-493a-af46-380e3f3d6574","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"submitter-timeout-g4hph","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:31:54.175Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"fe1339b8-daef-493a-af46-380e3f3d6574"}
{"level":"error","ts":"2026-11-03T15:31:54.175Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"fe1339b8-daef-493a-af46-380e3f3d6574","error":"mTLS secrets not ready: certificate ray-head-cert-submitter-timeout-g4hph not found: certificates.cert-manager.io \"ray-head-cert-submitter-timeout-g4hph\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:31:54.190Z","logger":"raycluster-resource","msg":"validate update","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:31:54.195Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"1e5faec2-d5e5-465a-9fdd-5b900de8e94b"}
{"level":"info","ts":"2026-11-03T15:31:54.195Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"1e5faec2-d5e5-465a-9fdd-5b900de8e94b"}
{"level":"info","ts":"2026-11-03T15:31:54.195Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"1e5faec2-d5e5-465a-9fdd-5b900de8e94b","1 head service found":"submitter-timeout-g4hph-head-svc"}
{"level":"info","ts":"2026-11-03T15:31:54.195Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"1e5faec2-d5e5-465a-9fdd-5b900de8e94b"}
{"level":"info","ts":"2026-11-03T15:31:54.245Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"579966cd-3d11-4b80-b381-611e15b25bb0","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:31:54.245Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"579966cd-3d11-4b80-b381-611e15b25bb0","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:31:54.245Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"579966cd-3d11-4b80-b381-611e15b25bb0","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:31:54.245Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"1e5faec2-d5e5-465a-9fdd-5b900de8e94b","error":"certificate ray-head-cert-submitter-timeout-g4hph not found: certificates.cert-manager.io \"ray-head-cert-submitter-timeout-g4hph\" not found"}
{"level":"info","ts":"2026-11-03T15:31:54.245Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"579966cd-3d11-4b80-b381-611e15b25bb0"}
{"level":"info","ts":"2026-11-03T15:31:54.245Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"579966cd-3d11-4b80-b381-611e15b25bb0","RayCluster":"submitter-timeout-g4hph","State":""}
{"level":"error","ts":"2026-11-03T15:31:54.245Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"1e5faec2-d5e5-465a-9fdd-5b900de8e94b","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-submitter-timeout-g4hph not found: certificates.cert-manager.io \"ray-head-cert-submitter-timeout-g4hph\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:31:54.245Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"1e5faec2-d5e5-465a-9fdd-5b900de8e94b","error":"mTLS secrets not ready: certificate ray-head-cert-submitter-timeout-g4hph not found: certificates.cert-manager.io \"ray-head-cert-submitter-timeout-g4hph\" not found"}
{"level":"info","ts":"2026-11-03T15:31:54.245Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"579966cd-3d11-4b80-b381-611e15b25bb0","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:31:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:31:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:31:54.245Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"1e5faec2-d5e5-465a-9fdd-5b900de8e94b","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"submitter-timeout-g4hph","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:31:54.245Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"1e5faec2-d5e5-465a-9fdd-5b900de8e94b","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"submitter-timeout-g4hph","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:31:54.245Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"1e5faec2-d5e5-465a-9fdd-5b900de8e94b","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"submitter-timeout-g4hph","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:31:54.245Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"1e5faec2-d5e5-465a-9fdd-5b900de8e94b"}
{"level":"error","ts":"2026-11-03T15:31:54.245Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"1e5faec2-d5e5-465a-9fdd-5b900de8e94b","error":"mTLS secrets not ready: certificate ray-head-cert-submitter-timeout-g4hph not found: certificates.cert-manager.io \"ray-head-cert-submitter-timeout-g4hph\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:31:54.286Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"912ca179-46d4-49e6-937c-d424c5b8bccc"}
{"level":"info","ts":"2026-11-03T15:31:54.286Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"912ca179-46d4-49e6-937c-d424c5b8bccc"}
{"level":"info","ts":"2026-11-03T15:31:54.286Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"912ca179-46d4-49e6-937c-d424c5b8bccc","1 head service found":"submitter-timeout-g4hph-head-svc"}
{"level":"info","ts":"2026-11-03T15:31:54.286Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"912ca179-46d4-49e6-937c-d424c5b8bccc"}
{"level":"info","ts":"2026-11-03T15:31:54.299Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"submitter-timeout-g4hph","reconcileID":"6add30f2-6d28-411f-a53a-56ab5b71bfa9","name":"submitter-timeout-g4hph-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-11-03T15:31:54.336Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"912ca179-46d4-49e6-937c-d424c5b8bccc","error":"certificate ray-head-cert-submitter-timeout-g4hph not found: certificates.cert-manager.io \"ray-head-cert-submitter-timeout-g4hph\" not found"}
{"level":"error","ts":"2026-11-03T15:31:54.336Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"912ca179-46d4-49e6-937c-d424c5b8bccc","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-submitter-timeout-g4hph not found: certificates.cert-manager.io \"ray-head-cert-submitter-timeout-g4hph\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:31:54.336Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"912ca179-46d4-49e6-937c-d424c5b8bccc","error":"mTLS secrets not ready: certificate ray-head-cert-submitter-timeout-g4hph not found: certificates.cert-manager.io \"ray-head-cert-submitter-timeout-g4hph\" not found"}
{"level":"info","ts":"2026-11-03T15:31:54.336Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"912ca179-46d4-49e6-937c-d424c5b8bccc","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"submitter-timeout-g4hph","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:31:54.336Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"912ca179-46d4-49e6-937c-d424c5b8bccc","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"submitter-timeout-g4hph","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:31:54.336Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"912ca179-46d4-49e6-937c-d424c5b8bccc","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"submitter-timeout-g4hph","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:31:54.336Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"912ca179-46d4-49e6-937c-d424c5b8bccc"}
{"level":"error","ts":"2026-11-03T15:31:54.336Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"912ca179-46d4-49e6-937c-d424c5b8bccc","error":"mTLS secrets not ready: certificate ray-head-cert-submitter-timeout-g4hph not found: certificates.cert-manager.io \"ray-head-cert-submitter-timeout-g4hph\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:31:54.354Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"submitter-timeout-g4hph","reconcileID":"6add30f2-6d28-411f-a53a-56ab5b71bfa9","name":"test-ns-kbx4w-submitter-timeout-g4hph","operation":"created"}
{"level":"info","ts":"2026-11-03T15:31:54.407Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"submitter-timeout-g4hph","reconcileID":"6add30f2-6d28-411f-a53a-56ab5b71bfa9","name":"kube-rbac-proxy-config-submitter-timeout-g4hph","operation":"created"}
{"level":"info","ts":"2026-11-03T15:31:54.417Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"2b9ca510-f8dc-4f4c-9be6-cd6851e8f6a0"}
{"level":"info","ts":"2026-11-03T15:31:54.417Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"2b9ca510-f8dc-4f4c-9be6-cd6851e8f6a0"}
{"level":"info","ts":"2026-11-03T15:31:54.417Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"2b9ca510-f8dc-4f4c-9be6-cd6851e8f6a0","1 head service found":"submitter-timeout-g4hph-head-svc"}
{"level":"info","ts":"2026-11-03T15:31:54.417Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"2b9ca510-f8dc-4f4c-9be6-cd6851e8f6a0"}
{"level":"error","ts":"2026-11-03T15:31:54.462Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"submitter-timeout-g4hph","reconcileID":"6add30f2-6d28-411f-a53a-56ab5b71bfa9","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"submitter-timeout-g4hph\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:167\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:31:54.462Z","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"submitter-timeout-g4hph","reconcileID":"6add30f2-6d28-411f-a53a-56ab5b71bfa9"}
{"level":"error","ts":"2026-11-03T15:31:54.462Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"submitter-timeout-g4hph","reconcileID":"6add30f2-6d28-411f-a53a-56ab5b71bfa9","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"submitter-timeout-g4hph\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:31:54.462Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"submitter-timeout-g4hph","reconcileID":"46a8ee1e-4240-4eb4-bbe4-bf7f97da721d","namespacedName":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:31:54.462Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"submitter-timeout-g4hph","reconcileID":"46a8ee1e-4240-4eb4-bbe4-bf7f97da721d","mode":"IntegratedOAuth","cluster":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:31:54.462Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"submitter-timeout-g4hph","reconcileID":"46a8ee1e-4240-4eb4-bbe4-bf7f97da721d","cluster":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:31:54.467Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"2b9ca510-f8dc-4f4c-9be6-cd6851e8f6a0","error":"certificate ray-head-cert-submitter-timeout-g4hph not found: certificates.cert-manager.io \"ray-head-cert-submitter-timeout-g4hph\" not found"}
{"level":"error","ts":"2026-11-03T15:31:54.467Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"2b9ca510-f8dc-4f4c-9be6-cd6851e8f6a0","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-submitter-timeout-g4hph not found: certificates.cert-manager.io \"ray-head-cert-submitter-timeout-g4hph\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:31:54.467Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"2b9ca510-f8dc-4f4c-9be6-cd6851e8f6a0","error":"mTLS secrets not ready: certificate ray-head-cert-submitter-timeout-g4hph not found: certificates.cert-manager.io \"ray-head-cert-submitter-timeout-g4hph\" not found"}
{"level":"info","ts":"2026-11-03T15:31:54.467Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"2b9ca510-f8dc-4f4c-9be6-cd6851e8f6a0","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"submitter-timeout-g4hph","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:31:54.467Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"2b9ca510-f8dc-4f4c-9be6-cd6851e8f6a0","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"submitter-timeout-g4hph","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:31:54.468Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"2b9ca510-f8dc-4f4c-9be6-cd6851e8f6a0","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"submitter-timeout-g4hph","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:31:54.468Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"2b9ca510-f8dc-4f4c-9be6-cd6851e8f6a0"}
{"level":"error","ts":"2026-11-03T15:31:54.468Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"2b9ca510-f8dc-4f4c-9be6-cd6851e8f6a0","error":"mTLS secrets not ready: certificate ray-head-cert-submitter-timeout-g4hph not found: certificates.cert-manager.io \"ray-head-cert-submitter-timeout-g4hph\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:31:54.525Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"submitter-timeout-g4hph","reconcileID":"46a8ee1e-4240-4eb4-bbe4-bf7f97da721d","cluster":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:31:54.525Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"20bbd3ad-9940-4772-8061-22c4a24c8d7d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:31:54.525Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"20bbd3ad-9940-4772-8061-22c4a24c8d7d","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:31:54.525Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"20bbd3ad-9940-4772-8061-22c4a24c8d7d","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:31:54.525Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"20bbd3ad-9940-4772-8061-22c4a24c8d7d"}
{"level":"info","ts":"2026-11-03T15:31:54.525Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"20bbd3ad-9940-4772-8061-22c4a24c8d7d","RayCluster":"submitter-timeout-g4hph","State":""}
{"level":"info","ts":"2026-11-03T15:31:54.525Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"20bbd3ad-9940-4772-8061-22c4a24c8d7d","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:31:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:31:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:31:54.525Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"20bbd3ad-9940-4772-8061-22c4a24c8d7d","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:31:54.588Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"528332a3-915a-4d2a-8614-8718ea7e2ad7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:31:54.588Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"528332a3-915a-4d2a-8614-8718ea7e2ad7","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:31:54.588Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"528332a3-915a-4d2a-8614-8718ea7e2ad7","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:31:54.588Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"528332a3-915a-4d2a-8614-8718ea7e2ad7"}
{"level":"info","ts":"2026-11-03T15:31:54.588Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"528332a3-915a-4d2a-8614-8718ea7e2ad7","RayCluster":"submitter-timeout-g4hph","State":""}
{"level":"info","ts":"2026-11-03T15:31:54.588Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"528332a3-915a-4d2a-8614-8718ea7e2ad7","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:31:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:31:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:31:54.628Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"4997d0e5-bbcf-4864-a956-7a613c0cc803"}
{"level":"info","ts":"2026-11-03T15:31:54.628Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"4997d0e5-bbcf-4864-a956-7a613c0cc803"}
{"level":"info","ts":"2026-11-03T15:31:54.628Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"4997d0e5-bbcf-4864-a956-7a613c0cc803","1 head service found":"submitter-timeout-g4hph-head-svc"}
{"level":"info","ts":"2026-11-03T15:31:54.628Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"4997d0e5-bbcf-4864-a956-7a613c0cc803"}
{"level":"info","ts":"2026-11-03T15:31:54.683Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"4997d0e5-bbcf-4864-a956-7a613c0cc803","error":"certificate ray-head-cert-submitter-timeout-g4hph is not ready at current generation 1"}
{"level":"error","ts":"2026-11-03T15:31:54.683Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"4997d0e5-bbcf-4864-a956-7a613c0cc803","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-submitter-timeout-g4hph is not ready at current generation 1","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:31:54.683Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"4997d0e5-bbcf-4864-a956-7a613c0cc803","error":"mTLS secrets not ready: certificate ray-head-cert-submitter-timeout-g4hph is not ready at current generation 1"}
{"level":"info","ts":"2026-11-03T15:31:54.683Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"4997d0e5-bbcf-4864-a956-7a613c0cc803","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"submitter-timeout-g4hph","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:31:54.683Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"4997d0e5-bbcf-4864-a956-7a613c0cc803","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"submitter-timeout-g4hph","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:31:54.683Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"4997d0e5-bbcf-4864-a956-7a613c0cc803","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"submitter-timeout-g4hph","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:31:54.683Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"4997d0e5-bbcf-4864-a956-7a613c0cc803"}
{"level":"error","ts":"2026-11-03T15:31:54.683Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"4997d0e5-bbcf-4864-a956-7a613c0cc803","error":"mTLS secrets not ready: certificate ray-head-cert-submitter-timeout-g4hph is not ready at current generation 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:31:54.790Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"7b6a5a40-ed39-42af-acd5-c8ec313ebe2f","certificate":"ray-head-cert-submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:31:54.790Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"7b6a5a40-ed39-42af-acd5-c8ec313ebe2f"}
{"level":"info","ts":"2026-11-03T15:31:54.790Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"986903b7-7a22-4d26-b0dd-91828c9e1f08","cluster":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:31:55.004Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"2edde5cd-f524-4018-b14b-1833b6d1a2e0"}
{"level":"info","ts":"2026-11-03T15:31:55.004Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"2edde5cd-f524-4018-b14b-1833b6d1a2e0"}
{"level":"info","ts":"2026-11-03T15:31:55.004Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"2edde5cd-f524-4018-b14b-1833b6d1a2e0","1 head service found":"submitter-timeout-g4hph-head-svc"}
{"level":"info","ts":"2026-11-03T15:31:55.004Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"2edde5cd-f524-4018-b14b-1833b6d1a2e0"}
{"level":"info","ts":"2026-11-03T15:31:55.055Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"2edde5cd-f524-4018-b14b-1833b6d1a2e0","error":"certificate ray-head-cert-submitter-timeout-g4hph is not ready at current generation 1"}
{"level":"error","ts":"2026-11-03T15:31:55.055Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"2edde5cd-f524-4018-b14b-1833b6d1a2e0","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-submitter-timeout-g4hph is not ready at current generation 1","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:31:55.055Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"2edde5cd-f524-4018-b14b-1833b6d1a2e0","error":"mTLS secrets not ready: certificate ray-head-cert-submitter-timeout-g4hph is not ready at current generation 1"}
{"level":"info","ts":"2026-11-03T15:31:55.055Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"2edde5cd-f524-4018-b14b-1833b6d1a2e0","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"submitter-timeout-g4hph","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:31:55.055Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"2edde5cd-f524-4018-b14b-1833b6d1a2e0","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"submitter-timeout-g4hph","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:31:55.055Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"2edde5cd-f524-4018-b14b-1833b6d1a2e0","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"submitter-timeout-g4hph","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:31:55.055Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"2edde5cd-f524-4018-b14b-1833b6d1a2e0"}
{"level":"error","ts":"2026-11-03T15:31:55.055Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"2edde5cd-f524-4018-b14b-1833b6d1a2e0","error":"mTLS secrets not ready: certificate ray-head-cert-submitter-timeout-g4hph is not ready at current generation 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:31:55.254Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"986903b7-7a22-4d26-b0dd-91828c9e1f08","certificate":"ray-head-cert-submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:31:55.254Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"986903b7-7a22-4d26-b0dd-91828c9e1f08"}
{"level":"info","ts":"2026-11-03T15:31:55.696Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"85862a9c-794b-4e1d-b5bd-4e3361185170"}
{"level":"info","ts":"2026-11-03T15:31:55.696Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"85862a9c-794b-4e1d-b5bd-4e3361185170"}
{"level":"info","ts":"2026-11-03T15:31:55.696Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"85862a9c-794b-4e1d-b5bd-4e3361185170","1 head service found":"submitter-timeout-g4hph-head-svc"}
{"level":"info","ts":"2026-11-03T15:31:55.696Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"85862a9c-794b-4e1d-b5bd-4e3361185170"}
{"level":"info","ts":"2026-11-03T15:31:55.747Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"85862a9c-794b-4e1d-b5bd-4e3361185170","error":"certificate ray-head-cert-submitter-timeout-g4hph is not ready at current generation 1"}
{"level":"error","ts":"2026-11-03T15:31:55.747Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"85862a9c-794b-4e1d-b5bd-4e3361185170","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-submitter-timeout-g4hph is not ready at current generation 1","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:31:55.747Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"85862a9c-794b-4e1d-b5bd-4e3361185170","error":"mTLS secrets not ready: certificate ray-head-cert-submitter-timeout-g4hph is not ready at current generation 1"}
{"level":"info","ts":"2026-11-03T15:31:55.747Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"85862a9c-794b-4e1d-b5bd-4e3361185170","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"submitter-timeout-g4hph","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:31:55.747Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"85862a9c-794b-4e1d-b5bd-4e3361185170","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"submitter-timeout-g4hph","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:31:55.747Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"85862a9c-794b-4e1d-b5bd-4e3361185170","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"submitter-timeout-g4hph","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:31:55.747Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"85862a9c-794b-4e1d-b5bd-4e3361185170"}
{"level":"error","ts":"2026-11-03T15:31:55.747Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"85862a9c-794b-4e1d-b5bd-4e3361185170","error":"mTLS secrets not ready: certificate ray-head-cert-submitter-timeout-g4hph is not ready at current generation 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:31:57.028Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"b5584282-db1e-4cb3-b1c9-f9ce9c9a18eb"}
{"level":"info","ts":"2026-11-03T15:31:57.028Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"b5584282-db1e-4cb3-b1c9-f9ce9c9a18eb"}
{"level":"info","ts":"2026-11-03T15:31:57.028Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"b5584282-db1e-4cb3-b1c9-f9ce9c9a18eb","1 head service found":"submitter-timeout-g4hph-head-svc"}
{"level":"info","ts":"2026-11-03T15:31:57.028Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"b5584282-db1e-4cb3-b1c9-f9ce9c9a18eb"}
{"level":"info","ts":"2026-11-03T15:31:57.130Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"b5584282-db1e-4cb3-b1c9-f9ce9c9a18eb","mode":"IntegratedOAuth","cluster":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:31:57.130Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"b5584282-db1e-4cb3-b1c9-f9ce9c9a18eb","cluster":"submitter-timeout-g4hph","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-11-03T15:31:57.130Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"b5584282-db1e-4cb3-b1c9-f9ce9c9a18eb","cluster":"submitter-timeout-g4hph","authType":"IntegratedOAuth","serviceAccount":"submitter-timeout-g4hph-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-11-03T15:31:57.130Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"b5584282-db1e-4cb3-b1c9-f9ce9c9a18eb","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"submitter-timeout-g4hph","ray.io/group":"headgroup","ray.io/identifier":"submitter-timeout-g4hph-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-11-03T15:31:57.130Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"b5584282-db1e-4cb3-b1c9-f9ce9c9a18eb","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-11-03T15:31:57.130Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"b5584282-db1e-4cb3-b1c9-f9ce9c9a18eb","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-11-03T15:31:57.130Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"b5584282-db1e-4cb3-b1c9-f9ce9c9a18eb","rayNodeType":"head","generatedCmd":"ulimit -n ${RAY_START_ULIMIT_OPEN_FILES:-65536}; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-11-03T15:31:57.130Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"b5584282-db1e-4cb3-b1c9-f9ce9c9a18eb","enabled":true}
{"level":"info","ts":"2026-11-03T15:31:57.262Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"b5584282-db1e-4cb3-b1c9-f9ce9c9a18eb","name":"submitter-timeout-g4hph-head-zjgxv"}
{"level":"info","ts":"2026-11-03T15:31:57.262Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"b5584282-db1e-4cb3-b1c9-f9ce9c9a18eb","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:31:57.262Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"b5584282-db1e-4cb3-b1c9-f9ce9c9a18eb","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:31:57.262Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"b5584282-db1e-4cb3-b1c9-f9ce9c9a18eb","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-11-03T15:31:57.262Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"b5584282-db1e-4cb3-b1c9-f9ce9c9a18eb","group":"small-group","indices":{}}
{"level":"info","ts":"2026-11-03T15:31:57.262Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"b5584282-db1e-4cb3-b1c9-f9ce9c9a18eb","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-11-03T15:31:57.363Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"b5584282-db1e-4cb3-b1c9-f9ce9c9a18eb","creating worker for group":"small-group","index":0,"total":1,"replicaIndex":0}
{"level":"info","ts":"2026-11-03T15:31:57.363Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"b5584282-db1e-4cb3-b1c9-f9ce9c9a18eb","nodeType":"worker","rayStartParams":{"address":"submitter-timeout-g4hph-head-svc.test-ns-kbx4w.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-11-03T15:31:57.363Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"b5584282-db1e-4cb3-b1c9-f9ce9c9a18eb","rayStartCmd":"ray start  --address=submitter-timeout-g4hph-head-svc.test-ns-kbx4w.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-11-03T15:31:57.364Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"b5584282-db1e-4cb3-b1c9-f9ce9c9a18eb","rayNodeType":"worker","generatedCmd":"ulimit -n ${RAY_START_ULIMIT_OPEN_FILES:-65536}; ray start  --address=submitter-timeout-g4hph-head-svc.test-ns-kbx4w.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-11-03T15:31:57.364Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"b5584282-db1e-4cb3-b1c9-f9ce9c9a18eb","enabled":true}
{"level":"info","ts":"2026-11-03T15:31:57.426Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"b5584282-db1e-4cb3-b1c9-f9ce9c9a18eb","name":"submitter-timeout-g4hph-small-group-worker-jsjx7"}
{"level":"info","ts":"2026-11-03T15:31:57.426Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"b5584282-db1e-4cb3-b1c9-f9ce9c9a18eb","podName":"submitter-timeout-g4hph-head-zjgxv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:31:57.426Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"b5584282-db1e-4cb3-b1c9-f9ce9c9a18eb","name":"submitter-timeout-g4hph","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:31:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:31:57Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:31:57.488Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"99771c1f-ad91-43d2-9a9a-debba5669cfd"}
{"level":"info","ts":"2026-11-03T15:31:57.488Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"99771c1f-ad91-43d2-9a9a-debba5669cfd"}
{"level":"info","ts":"2026-11-03T15:31:57.488Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"99771c1f-ad91-43d2-9a9a-debba5669cfd","1 head service found":"submitter-timeout-g4hph-head-svc"}
{"level":"info","ts":"2026-11-03T15:31:57.488Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"99771c1f-ad91-43d2-9a9a-debba5669cfd","Found 1 head Pod":"submitter-timeout-g4hph-head-zjgxv","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:31:57.488Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"99771c1f-ad91-43d2-9a9a-debba5669cfd","head Pod":"submitter-timeout-g4hph-head-zjgxv","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod submitter-timeout-g4hph-head-zjgxv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:31:57.488Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"99771c1f-ad91-43d2-9a9a-debba5669cfd","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:31:57.488Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"f0abdf8e-ee94-4e08-acb9-94ca3a21d1de","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:31:57.488Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"f0abdf8e-ee94-4e08-acb9-94ca3a21d1de","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:31:57.488Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"99771c1f-ad91-43d2-9a9a-debba5669cfd","worker Pod":"submitter-timeout-g4hph-small-group-worker-jsjx7","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod submitter-timeout-g4hph-small-group-worker-jsjx7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:31:57.488Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"99771c1f-ad91-43d2-9a9a-debba5669cfd","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:31:57.488Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"99771c1f-ad91-43d2-9a9a-debba5669cfd","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:31:57.488Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"99771c1f-ad91-43d2-9a9a-debba5669cfd","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:31:57.488Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"99771c1f-ad91-43d2-9a9a-debba5669cfd","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:31:57.488Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"f0abdf8e-ee94-4e08-acb9-94ca3a21d1de","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:31:57.488Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"99771c1f-ad91-43d2-9a9a-debba5669cfd","podName":"submitter-timeout-g4hph-head-zjgxv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:31:57.488Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"99771c1f-ad91-43d2-9a9a-debba5669cfd","podName":"submitter-timeout-g4hph-head-zjgxv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:31:57.488Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"f0abdf8e-ee94-4e08-acb9-94ca3a21d1de"}
{"level":"info","ts":"2026-11-03T15:31:57.488Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"f0abdf8e-ee94-4e08-acb9-94ca3a21d1de","RayCluster":"submitter-timeout-g4hph","State":""}
{"level":"info","ts":"2026-11-03T15:31:57.488Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"f0abdf8e-ee94-4e08-acb9-94ca3a21d1de","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:31:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:31:57Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:31:57.488Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"f0abdf8e-ee94-4e08-acb9-94ca3a21d1de","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:31:57.488Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"99771c1f-ad91-43d2-9a9a-debba5669cfd","name":"submitter-timeout-g4hph","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:31:53Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:31:57Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:31:57.488Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"fc7d71fa-4646-47d2-93df-a23aa2162169","cluster":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:31:57.543Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"99771c1f-ad91-43d2-9a9a-debba5669cfd","name":"submitter-timeout-g4hph","error":"Operation cannot be fulfilled on rayclusters.ray.io \"submitter-timeout-g4hph\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-kbx4w","name":"submitter-timeout-g4hph"}}
{"level":"info","ts":"2026-11-03T15:31:57.543Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"99771c1f-ad91-43d2-9a9a-debba5669cfd"}
{"level":"error","ts":"2026-11-03T15:31:57.543Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"99771c1f-ad91-43d2-9a9a-debba5669cfd","error":"Operation cannot be fulfilled on rayclusters.ray.io \"submitter-timeout-g4hph\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:31:57.544Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"7c168706-c188-4859-9460-7f28157b0d5c"}
{"level":"info","ts":"2026-11-03T15:31:57.544Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"7c168706-c188-4859-9460-7f28157b0d5c"}
{"level":"info","ts":"2026-11-03T15:31:57.544Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"7c168706-c188-4859-9460-7f28157b0d5c","1 head service found":"submitter-timeout-g4hph-head-svc"}
{"level":"info","ts":"2026-11-03T15:31:57.544Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"7c168706-c188-4859-9460-7f28157b0d5c","Found 1 head Pod":"submitter-timeout-g4hph-head-zjgxv","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:31:57.544Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"7c168706-c188-4859-9460-7f28157b0d5c","head Pod":"submitter-timeout-g4hph-head-zjgxv","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod submitter-timeout-g4hph-head-zjgxv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:31:57.544Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"7c168706-c188-4859-9460-7f28157b0d5c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:31:57.544Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"7c168706-c188-4859-9460-7f28157b0d5c","worker Pod":"submitter-timeout-g4hph-small-group-worker-jsjx7","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod submitter-timeout-g4hph-small-group-worker-jsjx7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:31:57.544Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"7c168706-c188-4859-9460-7f28157b0d5c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:31:57.544Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"7c168706-c188-4859-9460-7f28157b0d5c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:31:57.544Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"7c168706-c188-4859-9460-7f28157b0d5c","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:31:57.544Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"7c168706-c188-4859-9460-7f28157b0d5c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:31:57.544Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"7c168706-c188-4859-9460-7f28157b0d5c","podName":"submitter-timeout-g4hph-head-zjgxv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:31:57.544Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"7c168706-c188-4859-9460-7f28157b0d5c","podName":"submitter-timeout-g4hph-head-zjgxv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:31:57.544Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"7c168706-c188-4859-9460-7f28157b0d5c","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:31:57.544Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"7c168706-c188-4859-9460-7f28157b0d5c","seconds":300}
{"level":"info","ts":"2026-11-03T15:31:57.551Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"b002eaf0-0d95-4ccf-a2d8-e294fab8390c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:31:57.551Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"b002eaf0-0d95-4ccf-a2d8-e294fab8390c","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:31:57.552Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"b002eaf0-0d95-4ccf-a2d8-e294fab8390c","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:31:57.552Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"b002eaf0-0d95-4ccf-a2d8-e294fab8390c"}
{"level":"info","ts":"2026-11-03T15:31:57.552Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"b002eaf0-0d95-4ccf-a2d8-e294fab8390c","RayCluster":"submitter-timeout-g4hph","State":""}
{"level":"info","ts":"2026-11-03T15:31:57.552Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"b002eaf0-0d95-4ccf-a2d8-e294fab8390c","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:31:57Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:31:57Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:31:57.556Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"69e675d0-c671-4edc-92e3-698b3e4fabdc"}
{"level":"info","ts":"2026-11-03T15:31:57.556Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"69e675d0-c671-4edc-92e3-698b3e4fabdc"}
{"level":"info","ts":"2026-11-03T15:31:57.556Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"69e675d0-c671-4edc-92e3-698b3e4fabdc","1 head service found":"submitter-timeout-g4hph-head-svc"}
{"level":"info","ts":"2026-11-03T15:31:57.556Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"69e675d0-c671-4edc-92e3-698b3e4fabdc","Found 1 head Pod":"submitter-timeout-g4hph-head-zjgxv","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:31:57.556Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"69e675d0-c671-4edc-92e3-698b3e4fabdc","head Pod":"submitter-timeout-g4hph-head-zjgxv","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod submitter-timeout-g4hph-head-zjgxv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:31:57.556Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"69e675d0-c671-4edc-92e3-698b3e4fabdc","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:31:57.556Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"69e675d0-c671-4edc-92e3-698b3e4fabdc","worker Pod":"submitter-timeout-g4hph-small-group-worker-jsjx7","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod submitter-timeout-g4hph-small-group-worker-jsjx7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:31:57.556Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"69e675d0-c671-4edc-92e3-698b3e4fabdc","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:31:57.556Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"69e675d0-c671-4edc-92e3-698b3e4fabdc","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:31:57.556Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"69e675d0-c671-4edc-92e3-698b3e4fabdc","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:31:57.556Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"69e675d0-c671-4edc-92e3-698b3e4fabdc","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:31:57.556Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"69e675d0-c671-4edc-92e3-698b3e4fabdc","podName":"submitter-timeout-g4hph-head-zjgxv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:31:57.557Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"69e675d0-c671-4edc-92e3-698b3e4fabdc","podName":"submitter-timeout-g4hph-head-zjgxv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:31:57.557Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"69e675d0-c671-4edc-92e3-698b3e4fabdc","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:31:57.557Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"69e675d0-c671-4edc-92e3-698b3e4fabdc","seconds":300}
{"level":"info","ts":"2026-11-03T15:31:58.019Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"879da594-3dd8-412a-9fcc-588151180b61"}
{"level":"info","ts":"2026-11-03T15:31:58.019Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"879da594-3dd8-412a-9fcc-588151180b61"}
{"level":"info","ts":"2026-11-03T15:31:58.019Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"879da594-3dd8-412a-9fcc-588151180b61","1 head service found":"submitter-timeout-g4hph-head-svc"}
{"level":"info","ts":"2026-11-03T15:31:58.019Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"879da594-3dd8-412a-9fcc-588151180b61","Found 1 head Pod":"submitter-timeout-g4hph-head-zjgxv","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:31:58.019Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"879da594-3dd8-412a-9fcc-588151180b61","head Pod":"submitter-timeout-g4hph-head-zjgxv","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod submitter-timeout-g4hph-head-zjgxv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:31:58.019Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"879da594-3dd8-412a-9fcc-588151180b61","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:31:58.019Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"879da594-3dd8-412a-9fcc-588151180b61","worker Pod":"submitter-timeout-g4hph-small-group-worker-jsjx7","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod submitter-timeout-g4hph-small-group-worker-jsjx7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:31:58.019Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"879da594-3dd8-412a-9fcc-588151180b61","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:31:58.019Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"879da594-3dd8-412a-9fcc-588151180b61","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:31:58.019Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"879da594-3dd8-412a-9fcc-588151180b61","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:31:58.019Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"879da594-3dd8-412a-9fcc-588151180b61","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:31:58.019Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"879da594-3dd8-412a-9fcc-588151180b61","podName":"submitter-timeout-g4hph-head-zjgxv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:31:58.019Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"879da594-3dd8-412a-9fcc-588151180b61","podName":"submitter-timeout-g4hph-head-zjgxv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:31:58.020Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"879da594-3dd8-412a-9fcc-588151180b61","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:31:58.020Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"879da594-3dd8-412a-9fcc-588151180b61","seconds":300}
{"level":"info","ts":"2026-11-03T15:31:58.229Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"12c953b8-6b7d-4cde-b7b4-34e698c4d11f"}
{"level":"info","ts":"2026-11-03T15:31:58.229Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"12c953b8-6b7d-4cde-b7b4-34e698c4d11f"}
{"level":"info","ts":"2026-11-03T15:31:58.229Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"12c953b8-6b7d-4cde-b7b4-34e698c4d11f","1 head service found":"submitter-timeout-g4hph-head-svc"}
{"level":"info","ts":"2026-11-03T15:31:58.229Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"12c953b8-6b7d-4cde-b7b4-34e698c4d11f","Found 1 head Pod":"submitter-timeout-g4hph-head-zjgxv","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:31:58.229Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"12c953b8-6b7d-4cde-b7b4-34e698c4d11f","head Pod":"submitter-timeout-g4hph-head-zjgxv","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod submitter-timeout-g4hph-head-zjgxv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:31:58.229Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"12c953b8-6b7d-4cde-b7b4-34e698c4d11f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:31:58.229Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"12c953b8-6b7d-4cde-b7b4-34e698c4d11f","worker Pod":"submitter-timeout-g4hph-small-group-worker-jsjx7","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod submitter-timeout-g4hph-small-group-worker-jsjx7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:31:58.229Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"12c953b8-6b7d-4cde-b7b4-34e698c4d11f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:31:58.229Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"12c953b8-6b7d-4cde-b7b4-34e698c4d11f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:31:58.229Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"12c953b8-6b7d-4cde-b7b4-34e698c4d11f","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:31:58.229Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"12c953b8-6b7d-4cde-b7b4-34e698c4d11f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:31:58.229Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"12c953b8-6b7d-4cde-b7b4-34e698c4d11f","podName":"submitter-timeout-g4hph-head-zjgxv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:31:58.229Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"12c953b8-6b7d-4cde-b7b4-34e698c4d11f","podName":"submitter-timeout-g4hph-head-zjgxv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:31:58.229Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"12c953b8-6b7d-4cde-b7b4-34e698c4d11f","name":"submitter-timeout-g4hph","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:31:57Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:31:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:31:58.291Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"ca682c91-33a0-4c89-9276-bb550eed81da"}
{"level":"info","ts":"2026-11-03T15:31:58.291Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"ca682c91-33a0-4c89-9276-bb550eed81da"}
{"level":"info","ts":"2026-11-03T15:31:58.291Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"ca682c91-33a0-4c89-9276-bb550eed81da","1 head service found":"submitter-timeout-g4hph-head-svc"}
{"level":"info","ts":"2026-11-03T15:31:58.291Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"ca682c91-33a0-4c89-9276-bb550eed81da","Found 1 head Pod":"submitter-timeout-g4hph-head-zjgxv","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:31:58.291Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"ca682c91-33a0-4c89-9276-bb550eed81da","head Pod":"submitter-timeout-g4hph-head-zjgxv","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod submitter-timeout-g4hph-head-zjgxv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:31:58.291Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"ca682c91-33a0-4c89-9276-bb550eed81da","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:31:58.291Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"ca682c91-33a0-4c89-9276-bb550eed81da","worker Pod":"submitter-timeout-g4hph-small-group-worker-jsjx7","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod submitter-timeout-g4hph-small-group-worker-jsjx7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:31:58.291Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"ca682c91-33a0-4c89-9276-bb550eed81da","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:31:58.291Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"ca682c91-33a0-4c89-9276-bb550eed81da","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:31:58.291Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"ca682c91-33a0-4c89-9276-bb550eed81da","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:31:58.291Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"ca682c91-33a0-4c89-9276-bb550eed81da","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:31:58.291Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"ca682c91-33a0-4c89-9276-bb550eed81da","podName":"submitter-timeout-g4hph-head-zjgxv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:31:58.291Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"829aab1d-7184-4338-a8a7-843c6b50e5a8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:31:58.291Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"ca682c91-33a0-4c89-9276-bb550eed81da","podName":"submitter-timeout-g4hph-head-zjgxv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:31:58.291Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"829aab1d-7184-4338-a8a7-843c6b50e5a8","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:31:58.291Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"829aab1d-7184-4338-a8a7-843c6b50e5a8","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:31:58.291Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"ca682c91-33a0-4c89-9276-bb550eed81da","name":"submitter-timeout-g4hph","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:31:57Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:31:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:31:58.291Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"829aab1d-7184-4338-a8a7-843c6b50e5a8"}
{"level":"info","ts":"2026-11-03T15:31:58.291Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"829aab1d-7184-4338-a8a7-843c6b50e5a8","RayCluster":"submitter-timeout-g4hph","State":""}
{"level":"info","ts":"2026-11-03T15:31:58.291Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"829aab1d-7184-4338-a8a7-843c6b50e5a8","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:31:57Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:31:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:31:58.291Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"829aab1d-7184-4338-a8a7-843c6b50e5a8","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:31:58.318Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"a2da5081-a5bf-423f-968f-dd299cc2eb80","cluster":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:31:58.347Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"ca682c91-33a0-4c89-9276-bb550eed81da","name":"submitter-timeout-g4hph","error":"Operation cannot be fulfilled on rayclusters.ray.io \"submitter-timeout-g4hph\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-kbx4w","name":"submitter-timeout-g4hph"}}
{"level":"info","ts":"2026-11-03T15:31:58.347Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"ca682c91-33a0-4c89-9276-bb550eed81da"}
{"level":"error","ts":"2026-11-03T15:31:58.347Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"ca682c91-33a0-4c89-9276-bb550eed81da","error":"Operation cannot be fulfilled on rayclusters.ray.io \"submitter-timeout-g4hph\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:31:58.352Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"4ac305eb-c54e-4971-8a97-c7c4bfce43dc"}
{"level":"info","ts":"2026-11-03T15:31:58.352Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"4ac305eb-c54e-4971-8a97-c7c4bfce43dc"}
{"level":"info","ts":"2026-11-03T15:31:58.352Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"4ac305eb-c54e-4971-8a97-c7c4bfce43dc","1 head service found":"submitter-timeout-g4hph-head-svc"}
{"level":"info","ts":"2026-11-03T15:31:58.352Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"4ac305eb-c54e-4971-8a97-c7c4bfce43dc","Found 1 head Pod":"submitter-timeout-g4hph-head-zjgxv","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:31:58.352Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"4ac305eb-c54e-4971-8a97-c7c4bfce43dc","head Pod":"submitter-timeout-g4hph-head-zjgxv","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod submitter-timeout-g4hph-head-zjgxv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:31:58.352Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"4ac305eb-c54e-4971-8a97-c7c4bfce43dc","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:31:58.352Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"4ac305eb-c54e-4971-8a97-c7c4bfce43dc","worker Pod":"submitter-timeout-g4hph-small-group-worker-jsjx7","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod submitter-timeout-g4hph-small-group-worker-jsjx7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:31:58.352Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"4ac305eb-c54e-4971-8a97-c7c4bfce43dc","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:31:58.352Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"4ac305eb-c54e-4971-8a97-c7c4bfce43dc","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:31:58.352Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"4ac305eb-c54e-4971-8a97-c7c4bfce43dc","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:31:58.352Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"4ac305eb-c54e-4971-8a97-c7c4bfce43dc","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:31:58.352Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"4ac305eb-c54e-4971-8a97-c7c4bfce43dc","podName":"submitter-timeout-g4hph-small-group-worker-jsjx7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:31:58.352Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"4ac305eb-c54e-4971-8a97-c7c4bfce43dc","podName":"submitter-timeout-g4hph-small-group-worker-jsjx7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:31:58.352Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"4ac305eb-c54e-4971-8a97-c7c4bfce43dc","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:31:58.352Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"4ac305eb-c54e-4971-8a97-c7c4bfce43dc","seconds":300}
{"level":"info","ts":"2026-11-03T15:31:58.353Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"adb440f4-7048-438c-a923-3adb27bf4fd3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:31:58.353Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"adb440f4-7048-438c-a923-3adb27bf4fd3","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:31:58.353Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"adb440f4-7048-438c-a923-3adb27bf4fd3","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:31:58.353Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"adb440f4-7048-438c-a923-3adb27bf4fd3"}
{"level":"info","ts":"2026-11-03T15:31:58.353Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"adb440f4-7048-438c-a923-3adb27bf4fd3","RayCluster":"submitter-timeout-g4hph","State":""}
{"level":"info","ts":"2026-11-03T15:31:58.353Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"adb440f4-7048-438c-a923-3adb27bf4fd3","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:31:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:31:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:31:58.627Z","logger":"controllers.RayClusterMTLS","msg":"Updating head certificate","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"a2da5081-a5bf-423f-968f-dd299cc2eb80","certificate":"ray-head-cert-submitter-timeout-g4hph","ipAddresses":["10.134.0.50","127.0.0.1"]}
{"level":"info","ts":"2026-11-03T15:31:59.028Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"d863caf4-02b7-429a-b764-ac57384576e0"}
{"level":"info","ts":"2026-11-03T15:31:59.028Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"d863caf4-02b7-429a-b764-ac57384576e0"}
{"level":"info","ts":"2026-11-03T15:31:59.028Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"d863caf4-02b7-429a-b764-ac57384576e0","1 head service found":"submitter-timeout-g4hph-head-svc"}
{"level":"info","ts":"2026-11-03T15:31:59.028Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"d863caf4-02b7-429a-b764-ac57384576e0","Found 1 head Pod":"submitter-timeout-g4hph-head-zjgxv","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:31:59.028Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"d863caf4-02b7-429a-b764-ac57384576e0","head Pod":"submitter-timeout-g4hph-head-zjgxv","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod submitter-timeout-g4hph-head-zjgxv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:31:59.028Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"d863caf4-02b7-429a-b764-ac57384576e0","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:31:59.028Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"d863caf4-02b7-429a-b764-ac57384576e0","worker Pod":"submitter-timeout-g4hph-small-group-worker-jsjx7","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod submitter-timeout-g4hph-small-group-worker-jsjx7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:31:59.028Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"d863caf4-02b7-429a-b764-ac57384576e0","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:31:59.028Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"d863caf4-02b7-429a-b764-ac57384576e0","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:31:59.028Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"d863caf4-02b7-429a-b764-ac57384576e0","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:31:59.028Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"d863caf4-02b7-429a-b764-ac57384576e0","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:31:59.029Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"d863caf4-02b7-429a-b764-ac57384576e0","podName":"submitter-timeout-g4hph-head-zjgxv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:31:59.029Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"d863caf4-02b7-429a-b764-ac57384576e0","podName":"submitter-timeout-g4hph-head-zjgxv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:31:59.029Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"d863caf4-02b7-429a-b764-ac57384576e0","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:31:59.029Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"d863caf4-02b7-429a-b764-ac57384576e0","seconds":300}
{"level":"info","ts":"2026-11-03T15:31:59.041Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"a2da5081-a5bf-423f-968f-dd299cc2eb80","certificate":"ray-head-cert-submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:31:59.041Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"a2da5081-a5bf-423f-968f-dd299cc2eb80"}
{"level":"info","ts":"2026-11-03T15:32:01.355Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"2432962a-dd64-4e1a-b0e8-d38ddc3b1541","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:32:01.355Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"2432962a-dd64-4e1a-b0e8-d38ddc3b1541","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:32:01.355Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"2432962a-dd64-4e1a-b0e8-d38ddc3b1541","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:32:01.355Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"2432962a-dd64-4e1a-b0e8-d38ddc3b1541"}
{"level":"info","ts":"2026-11-03T15:32:01.355Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"2432962a-dd64-4e1a-b0e8-d38ddc3b1541","RayCluster":"submitter-timeout-g4hph","State":""}
{"level":"info","ts":"2026-11-03T15:32:01.355Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"2432962a-dd64-4e1a-b0e8-d38ddc3b1541","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:31:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:31:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:32:04.356Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"16a9cf6d-a2bc-402f-9315-e8a40c573f3b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:32:04.356Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"16a9cf6d-a2bc-402f-9315-e8a40c573f3b","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:32:04.356Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"16a9cf6d-a2bc-402f-9315-e8a40c573f3b","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:32:04.356Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"16a9cf6d-a2bc-402f-9315-e8a40c573f3b"}
{"level":"info","ts":"2026-11-03T15:32:04.356Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"16a9cf6d-a2bc-402f-9315-e8a40c573f3b","RayCluster":"submitter-timeout-g4hph","State":""}
{"level":"info","ts":"2026-11-03T15:32:04.356Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"16a9cf6d-a2bc-402f-9315-e8a40c573f3b","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:31:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:31:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:32:07.357Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"cff24ed9-0d28-41f1-ad61-6b0cad3f7952","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:32:07.357Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"cff24ed9-0d28-41f1-ad61-6b0cad3f7952","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:32:07.357Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"cff24ed9-0d28-41f1-ad61-6b0cad3f7952","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:32:07.357Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"cff24ed9-0d28-41f1-ad61-6b0cad3f7952"}
{"level":"info","ts":"2026-11-03T15:32:07.357Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"cff24ed9-0d28-41f1-ad61-6b0cad3f7952","RayCluster":"submitter-timeout-g4hph","State":""}
{"level":"info","ts":"2026-11-03T15:32:07.357Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"cff24ed9-0d28-41f1-ad61-6b0cad3f7952","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:31:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:31:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:32:10.358Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"198b5aea-b9fe-43e4-875a-fa91269a1ae4","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:32:10.358Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"198b5aea-b9fe-43e4-875a-fa91269a1ae4","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:32:10.358Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"198b5aea-b9fe-43e4-875a-fa91269a1ae4","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:32:10.358Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"198b5aea-b9fe-43e4-875a-fa91269a1ae4"}
{"level":"info","ts":"2026-11-03T15:32:10.358Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"198b5aea-b9fe-43e4-875a-fa91269a1ae4","RayCluster":"submitter-timeout-g4hph","State":""}
{"level":"info","ts":"2026-11-03T15:32:10.358Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"198b5aea-b9fe-43e4-875a-fa91269a1ae4","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:31:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:31:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:32:13.359Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"4059c154-1fcb-4da8-888f-4142f198a1ae","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:32:13.359Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"4059c154-1fcb-4da8-888f-4142f198a1ae","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:32:13.359Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"4059c154-1fcb-4da8-888f-4142f198a1ae","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:32:13.359Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"4059c154-1fcb-4da8-888f-4142f198a1ae"}
{"level":"info","ts":"2026-11-03T15:32:13.359Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"4059c154-1fcb-4da8-888f-4142f198a1ae","RayCluster":"submitter-timeout-g4hph","State":""}
{"level":"info","ts":"2026-11-03T15:32:13.359Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"4059c154-1fcb-4da8-888f-4142f198a1ae","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:31:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:31:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:32:16.360Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"ce9aa979-a119-4e33-92dd-12ad0e484dc3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:32:16.360Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"ce9aa979-a119-4e33-92dd-12ad0e484dc3","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:32:16.360Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"ce9aa979-a119-4e33-92dd-12ad0e484dc3","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:32:16.361Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"ce9aa979-a119-4e33-92dd-12ad0e484dc3"}
{"level":"info","ts":"2026-11-03T15:32:16.361Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"ce9aa979-a119-4e33-92dd-12ad0e484dc3","RayCluster":"submitter-timeout-g4hph","State":""}
{"level":"info","ts":"2026-11-03T15:32:16.361Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"ce9aa979-a119-4e33-92dd-12ad0e484dc3","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:31:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:31:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:32:19.361Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"5e5ced66-0185-4991-bb07-608cb2c07bed","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:32:19.361Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"5e5ced66-0185-4991-bb07-608cb2c07bed","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:32:19.361Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"5e5ced66-0185-4991-bb07-608cb2c07bed","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:32:19.361Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"5e5ced66-0185-4991-bb07-608cb2c07bed"}
{"level":"info","ts":"2026-11-03T15:32:19.361Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"5e5ced66-0185-4991-bb07-608cb2c07bed","RayCluster":"submitter-timeout-g4hph","State":""}
{"level":"info","ts":"2026-11-03T15:32:19.361Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"5e5ced66-0185-4991-bb07-608cb2c07bed","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:31:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:31:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:32:22.362Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"1bccc7cc-7510-464b-b932-6ead1693b050","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:32:22.362Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"1bccc7cc-7510-464b-b932-6ead1693b050","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:32:22.362Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"1bccc7cc-7510-464b-b932-6ead1693b050","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:32:22.362Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"1bccc7cc-7510-464b-b932-6ead1693b050"}
{"level":"info","ts":"2026-11-03T15:32:22.362Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"1bccc7cc-7510-464b-b932-6ead1693b050","RayCluster":"submitter-timeout-g4hph","State":""}
{"level":"info","ts":"2026-11-03T15:32:22.362Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"1bccc7cc-7510-464b-b932-6ead1693b050","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:31:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:31:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:32:25.363Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"fc245a72-d8f5-440f-bc11-97671587457c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:32:25.363Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"fc245a72-d8f5-440f-bc11-97671587457c","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:32:25.363Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"fc245a72-d8f5-440f-bc11-97671587457c","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:32:25.363Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"fc245a72-d8f5-440f-bc11-97671587457c"}
{"level":"info","ts":"2026-11-03T15:32:25.363Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"fc245a72-d8f5-440f-bc11-97671587457c","RayCluster":"submitter-timeout-g4hph","State":""}
{"level":"info","ts":"2026-11-03T15:32:25.363Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"fc245a72-d8f5-440f-bc11-97671587457c","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:31:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:31:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:32:28.364Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"88d85d42-14dd-44f9-9667-332c48e90c71","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:32:28.364Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"88d85d42-14dd-44f9-9667-332c48e90c71","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:32:28.364Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"88d85d42-14dd-44f9-9667-332c48e90c71","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:32:28.364Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"88d85d42-14dd-44f9-9667-332c48e90c71"}
{"level":"info","ts":"2026-11-03T15:32:28.364Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"88d85d42-14dd-44f9-9667-332c48e90c71","RayCluster":"submitter-timeout-g4hph","State":""}
{"level":"info","ts":"2026-11-03T15:32:28.364Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"88d85d42-14dd-44f9-9667-332c48e90c71","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:31:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:31:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:32:29.041Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"435d9f0c-9313-4664-a20d-28a1da45a8e9","cluster":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:32:29.513Z","logger":"controllers.RayClusterMTLS","msg":"Updating worker certificate","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"435d9f0c-9313-4664-a20d-28a1da45a8e9","certificate":"ray-worker-cert-submitter-timeout-g4hph","ipAddresses":["10.132.0.45","127.0.0.1"]}
{"level":"info","ts":"2026-11-03T15:32:29.830Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"435d9f0c-9313-4664-a20d-28a1da45a8e9","certificate":"ray-worker-cert-submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:32:29.830Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"435d9f0c-9313-4664-a20d-28a1da45a8e9"}
{"level":"info","ts":"2026-11-03T15:32:31.364Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"7e2451f2-577f-43d5-9807-4c4298abfe72","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:32:31.364Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"7e2451f2-577f-43d5-9807-4c4298abfe72","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:32:31.364Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"7e2451f2-577f-43d5-9807-4c4298abfe72","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:32:31.364Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"7e2451f2-577f-43d5-9807-4c4298abfe72"}
{"level":"info","ts":"2026-11-03T15:32:31.364Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"7e2451f2-577f-43d5-9807-4c4298abfe72","RayCluster":"submitter-timeout-g4hph","State":""}
{"level":"info","ts":"2026-11-03T15:32:31.364Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"7e2451f2-577f-43d5-9807-4c4298abfe72","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:31:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:31:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:32:34.366Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"2dbc3782-d4b1-462d-b370-3ab71d4c3e1d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:32:34.366Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"2dbc3782-d4b1-462d-b370-3ab71d4c3e1d","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:32:34.366Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"2dbc3782-d4b1-462d-b370-3ab71d4c3e1d","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:32:34.366Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"2dbc3782-d4b1-462d-b370-3ab71d4c3e1d"}
{"level":"info","ts":"2026-11-03T15:32:34.366Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"2dbc3782-d4b1-462d-b370-3ab71d4c3e1d","RayCluster":"submitter-timeout-g4hph","State":""}
{"level":"info","ts":"2026-11-03T15:32:34.366Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"2dbc3782-d4b1-462d-b370-3ab71d4c3e1d","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:31:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:31:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:32:37.367Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"e9ee4fc2-67a1-48b0-8839-c470745ec49d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:32:37.367Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"e9ee4fc2-67a1-48b0-8839-c470745ec49d","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:32:37.367Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"e9ee4fc2-67a1-48b0-8839-c470745ec49d","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:32:37.367Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"e9ee4fc2-67a1-48b0-8839-c470745ec49d"}
{"level":"info","ts":"2026-11-03T15:32:37.367Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"e9ee4fc2-67a1-48b0-8839-c470745ec49d","RayCluster":"submitter-timeout-g4hph","State":""}
{"level":"info","ts":"2026-11-03T15:32:37.367Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"e9ee4fc2-67a1-48b0-8839-c470745ec49d","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:31:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:31:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:32:40.368Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"54bb842c-91c6-4e08-8678-970eae79146b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:32:40.368Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"54bb842c-91c6-4e08-8678-970eae79146b","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:32:40.368Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"54bb842c-91c6-4e08-8678-970eae79146b","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:32:40.369Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"54bb842c-91c6-4e08-8678-970eae79146b"}
{"level":"info","ts":"2026-11-03T15:32:40.369Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"54bb842c-91c6-4e08-8678-970eae79146b","RayCluster":"submitter-timeout-g4hph","State":""}
{"level":"info","ts":"2026-11-03T15:32:40.369Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"54bb842c-91c6-4e08-8678-970eae79146b","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:31:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:31:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:32:40.608Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"9e9085a1-252a-409a-af95-729645764de2","cluster":"long-running-hvml9"}
{"level":"info","ts":"2026-11-03T15:32:43.369Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"1091ddb4-8501-4f05-bcf5-5a77a01efe18","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:32:43.369Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"1091ddb4-8501-4f05-bcf5-5a77a01efe18","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:32:43.369Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"1091ddb4-8501-4f05-bcf5-5a77a01efe18","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:32:43.369Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"1091ddb4-8501-4f05-bcf5-5a77a01efe18"}
{"level":"info","ts":"2026-11-03T15:32:43.369Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"1091ddb4-8501-4f05-bcf5-5a77a01efe18","RayCluster":"submitter-timeout-g4hph","State":""}
{"level":"info","ts":"2026-11-03T15:32:43.369Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"1091ddb4-8501-4f05-bcf5-5a77a01efe18","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:31:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:31:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:32:46.370Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"60313145-7a0e-4cfa-bf9f-d9232cace765","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:32:46.370Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"60313145-7a0e-4cfa-bf9f-d9232cace765","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:32:46.370Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"60313145-7a0e-4cfa-bf9f-d9232cace765","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:32:46.370Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"60313145-7a0e-4cfa-bf9f-d9232cace765"}
{"level":"info","ts":"2026-11-03T15:32:46.370Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"60313145-7a0e-4cfa-bf9f-d9232cace765","RayCluster":"submitter-timeout-g4hph","State":""}
{"level":"info","ts":"2026-11-03T15:32:46.370Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"60313145-7a0e-4cfa-bf9f-d9232cace765","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:31:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:31:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:32:49.371Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"7186b793-13e7-4446-a729-3e03c59bd0b4","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:32:49.371Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"7186b793-13e7-4446-a729-3e03c59bd0b4","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:32:49.371Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"7186b793-13e7-4446-a729-3e03c59bd0b4","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:32:49.371Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"7186b793-13e7-4446-a729-3e03c59bd0b4"}
{"level":"info","ts":"2026-11-03T15:32:49.371Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"7186b793-13e7-4446-a729-3e03c59bd0b4","RayCluster":"submitter-timeout-g4hph","State":""}
{"level":"info","ts":"2026-11-03T15:32:49.371Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"7186b793-13e7-4446-a729-3e03c59bd0b4","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:31:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:31:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:32:52.372Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"ad28b55c-99bf-4a4b-89d7-127c2a1fb53f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:32:52.372Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"ad28b55c-99bf-4a4b-89d7-127c2a1fb53f","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:32:52.372Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"ad28b55c-99bf-4a4b-89d7-127c2a1fb53f","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:32:52.372Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"ad28b55c-99bf-4a4b-89d7-127c2a1fb53f"}
{"level":"info","ts":"2026-11-03T15:32:52.372Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"ad28b55c-99bf-4a4b-89d7-127c2a1fb53f","RayCluster":"submitter-timeout-g4hph","State":""}
{"level":"info","ts":"2026-11-03T15:32:52.372Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"ad28b55c-99bf-4a4b-89d7-127c2a1fb53f","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:31:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:31:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:32:55.373Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"d22ef857-8265-4e45-bb23-c5255369af5e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:32:55.373Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"d22ef857-8265-4e45-bb23-c5255369af5e","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:32:55.373Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"d22ef857-8265-4e45-bb23-c5255369af5e","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:32:55.373Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"d22ef857-8265-4e45-bb23-c5255369af5e"}
{"level":"info","ts":"2026-11-03T15:32:55.373Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"d22ef857-8265-4e45-bb23-c5255369af5e","RayCluster":"submitter-timeout-g4hph","State":""}
{"level":"info","ts":"2026-11-03T15:32:55.373Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"d22ef857-8265-4e45-bb23-c5255369af5e","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:31:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:31:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:32:58.374Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"2af3aa44-c7c5-4ced-bc23-1f45ad9cc2ce","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:32:58.374Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"2af3aa44-c7c5-4ced-bc23-1f45ad9cc2ce","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:32:58.374Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"2af3aa44-c7c5-4ced-bc23-1f45ad9cc2ce","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:32:58.375Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"2af3aa44-c7c5-4ced-bc23-1f45ad9cc2ce"}
{"level":"info","ts":"2026-11-03T15:32:58.375Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"2af3aa44-c7c5-4ced-bc23-1f45ad9cc2ce","RayCluster":"submitter-timeout-g4hph","State":""}
{"level":"info","ts":"2026-11-03T15:32:58.375Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"2af3aa44-c7c5-4ced-bc23-1f45ad9cc2ce","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:31:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:31:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:32:59.831Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"28c194e6-c5a1-4a83-bbe5-22ecc2f04af0","cluster":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:33:01.375Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"ff4505a5-8d28-4a04-8f37-a272976db0b9","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:33:01.375Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"ff4505a5-8d28-4a04-8f37-a272976db0b9","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:33:01.375Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"ff4505a5-8d28-4a04-8f37-a272976db0b9","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:33:01.375Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"ff4505a5-8d28-4a04-8f37-a272976db0b9"}
{"level":"info","ts":"2026-11-03T15:33:01.376Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"ff4505a5-8d28-4a04-8f37-a272976db0b9","RayCluster":"submitter-timeout-g4hph","State":""}
{"level":"info","ts":"2026-11-03T15:33:01.376Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"ff4505a5-8d28-4a04-8f37-a272976db0b9","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:31:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:31:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:33:04.376Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"909068e5-cc3c-4ecf-b4a5-1c62b8987fbf","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:33:04.376Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"909068e5-cc3c-4ecf-b4a5-1c62b8987fbf","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:33:04.376Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"909068e5-cc3c-4ecf-b4a5-1c62b8987fbf","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:33:04.377Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"909068e5-cc3c-4ecf-b4a5-1c62b8987fbf"}
{"level":"info","ts":"2026-11-03T15:33:04.377Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"909068e5-cc3c-4ecf-b4a5-1c62b8987fbf","RayCluster":"submitter-timeout-g4hph","State":""}
{"level":"info","ts":"2026-11-03T15:33:04.377Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"909068e5-cc3c-4ecf-b4a5-1c62b8987fbf","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:31:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:31:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:33:07.378Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"483ad0d7-946e-4da4-a2f1-15394cdf43c1","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:33:07.378Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"483ad0d7-946e-4da4-a2f1-15394cdf43c1","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:33:07.378Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"483ad0d7-946e-4da4-a2f1-15394cdf43c1","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:33:07.378Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"483ad0d7-946e-4da4-a2f1-15394cdf43c1"}
{"level":"info","ts":"2026-11-03T15:33:07.378Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"483ad0d7-946e-4da4-a2f1-15394cdf43c1","RayCluster":"submitter-timeout-g4hph","State":""}
{"level":"info","ts":"2026-11-03T15:33:07.378Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"483ad0d7-946e-4da4-a2f1-15394cdf43c1","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:31:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:31:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:33:08.628Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"10a61da2-25cb-4c60-ad2f-d1ec32a85d05"}
{"level":"info","ts":"2026-11-03T15:33:08.628Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"10a61da2-25cb-4c60-ad2f-d1ec32a85d05"}
{"level":"info","ts":"2026-11-03T15:33:08.628Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"10a61da2-25cb-4c60-ad2f-d1ec32a85d05","1 head service found":"submitter-timeout-g4hph-head-svc"}
{"level":"info","ts":"2026-11-03T15:33:08.628Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"10a61da2-25cb-4c60-ad2f-d1ec32a85d05","Found 1 head Pod":"submitter-timeout-g4hph-head-zjgxv","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:33:08.628Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"10a61da2-25cb-4c60-ad2f-d1ec32a85d05","head Pod":"submitter-timeout-g4hph-head-zjgxv","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod submitter-timeout-g4hph-head-zjgxv. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:33:08.628Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"10a61da2-25cb-4c60-ad2f-d1ec32a85d05","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:33:08.628Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"10a61da2-25cb-4c60-ad2f-d1ec32a85d05","worker Pod":"submitter-timeout-g4hph-small-group-worker-jsjx7","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod submitter-timeout-g4hph-small-group-worker-jsjx7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:33:08.628Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"10a61da2-25cb-4c60-ad2f-d1ec32a85d05","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:33:08.628Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"10a61da2-25cb-4c60-ad2f-d1ec32a85d05","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:33:08.628Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"10a61da2-25cb-4c60-ad2f-d1ec32a85d05","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:33:08.628Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"10a61da2-25cb-4c60-ad2f-d1ec32a85d05","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:33:08.628Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"10a61da2-25cb-4c60-ad2f-d1ec32a85d05","podName":"submitter-timeout-g4hph-head-zjgxv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:33:08.628Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"10a61da2-25cb-4c60-ad2f-d1ec32a85d05","podName":"submitter-timeout-g4hph-head-zjgxv","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:33:08.628Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"10a61da2-25cb-4c60-ad2f-d1ec32a85d05","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:33:08.628Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"10a61da2-25cb-4c60-ad2f-d1ec32a85d05","seconds":300}
{"level":"info","ts":"2026-11-03T15:33:09.609Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"a50ea476-0f01-49fd-a626-f270d84c0bfd"}
{"level":"info","ts":"2026-11-03T15:33:09.609Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"a50ea476-0f01-49fd-a626-f270d84c0bfd"}
{"level":"info","ts":"2026-11-03T15:33:09.609Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"a50ea476-0f01-49fd-a626-f270d84c0bfd","1 head service found":"submitter-timeout-g4hph-head-svc"}
{"level":"info","ts":"2026-11-03T15:33:09.609Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"a50ea476-0f01-49fd-a626-f270d84c0bfd","Found 1 head Pod":"submitter-timeout-g4hph-head-zjgxv","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:33:09.609Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"a50ea476-0f01-49fd-a626-f270d84c0bfd","head Pod":"submitter-timeout-g4hph-head-zjgxv","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod submitter-timeout-g4hph-head-zjgxv. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:33:09.609Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"a50ea476-0f01-49fd-a626-f270d84c0bfd","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:33:09.609Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"a50ea476-0f01-49fd-a626-f270d84c0bfd","worker Pod":"submitter-timeout-g4hph-small-group-worker-jsjx7","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod submitter-timeout-g4hph-small-group-worker-jsjx7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:33:09.609Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"a50ea476-0f01-49fd-a626-f270d84c0bfd","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:33:09.609Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"a50ea476-0f01-49fd-a626-f270d84c0bfd","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:33:09.609Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"a50ea476-0f01-49fd-a626-f270d84c0bfd","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:33:09.609Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"a50ea476-0f01-49fd-a626-f270d84c0bfd","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:33:09.609Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"a50ea476-0f01-49fd-a626-f270d84c0bfd","podName":"submitter-timeout-g4hph-head-zjgxv","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:31:57Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:33:09.609Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"a50ea476-0f01-49fd-a626-f270d84c0bfd","podName":"submitter-timeout-g4hph-head-zjgxv","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:31:57Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:33:09.609Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"a50ea476-0f01-49fd-a626-f270d84c0bfd","name":"submitter-timeout-g4hph","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:31:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:33:09Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:33:09.676Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"3182450d-c076-4419-bbe4-0275cacead7b","cluster":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:33:09.676Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"444e6e33-7c21-49d5-a50b-cc1668954817","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:33:09.676Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"444e6e33-7c21-49d5-a50b-cc1668954817","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:33:09.676Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"444e6e33-7c21-49d5-a50b-cc1668954817","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:33:09.676Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"444e6e33-7c21-49d5-a50b-cc1668954817"}
{"level":"info","ts":"2026-11-03T15:33:09.676Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"444e6e33-7c21-49d5-a50b-cc1668954817","RayCluster":"submitter-timeout-g4hph","State":""}
{"level":"info","ts":"2026-11-03T15:33:09.676Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"444e6e33-7c21-49d5-a50b-cc1668954817","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:31:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:33:09Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:33:09.677Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"444e6e33-7c21-49d5-a50b-cc1668954817","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:33:09.739Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"930b6ffd-6de2-4093-a194-4d94c99cb5a7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:33:09.739Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"930b6ffd-6de2-4093-a194-4d94c99cb5a7","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:33:09.739Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"930b6ffd-6de2-4093-a194-4d94c99cb5a7","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:33:09.740Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"930b6ffd-6de2-4093-a194-4d94c99cb5a7"}
{"level":"info","ts":"2026-11-03T15:33:09.740Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"930b6ffd-6de2-4093-a194-4d94c99cb5a7","RayCluster":"submitter-timeout-g4hph","State":""}
{"level":"info","ts":"2026-11-03T15:33:09.740Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"930b6ffd-6de2-4093-a194-4d94c99cb5a7","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:33:09Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:33:09Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:33:11.675Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"d5cb94b6-85e2-4112-9551-589ffc4f1677"}
{"level":"info","ts":"2026-11-03T15:33:11.676Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"d5cb94b6-85e2-4112-9551-589ffc4f1677"}
{"level":"info","ts":"2026-11-03T15:33:11.676Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"d5cb94b6-85e2-4112-9551-589ffc4f1677","1 head service found":"submitter-timeout-g4hph-head-svc"}
{"level":"info","ts":"2026-11-03T15:33:11.676Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"d5cb94b6-85e2-4112-9551-589ffc4f1677","Found 1 head Pod":"submitter-timeout-g4hph-head-zjgxv","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:33:11.676Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"d5cb94b6-85e2-4112-9551-589ffc4f1677","head Pod":"submitter-timeout-g4hph-head-zjgxv","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod submitter-timeout-g4hph-head-zjgxv. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:33:11.676Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"d5cb94b6-85e2-4112-9551-589ffc4f1677","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:33:11.676Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"d5cb94b6-85e2-4112-9551-589ffc4f1677","worker Pod":"submitter-timeout-g4hph-small-group-worker-jsjx7","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod submitter-timeout-g4hph-small-group-worker-jsjx7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:33:11.676Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"d5cb94b6-85e2-4112-9551-589ffc4f1677","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:33:11.676Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"d5cb94b6-85e2-4112-9551-589ffc4f1677","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:33:11.676Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"d5cb94b6-85e2-4112-9551-589ffc4f1677","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:33:11.676Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"d5cb94b6-85e2-4112-9551-589ffc4f1677","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:33:11.676Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"d5cb94b6-85e2-4112-9551-589ffc4f1677","podName":"submitter-timeout-g4hph-head-zjgxv","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:31:57Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:33:11.677Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"d5cb94b6-85e2-4112-9551-589ffc4f1677","podName":"submitter-timeout-g4hph-head-zjgxv","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:31:57Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:33:11.677Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"d5cb94b6-85e2-4112-9551-589ffc4f1677","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:33:11.677Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"d5cb94b6-85e2-4112-9551-589ffc4f1677","seconds":300}
{"level":"info","ts":"2026-11-03T15:33:12.741Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"1d37ce18-3b44-4cd0-9bc6-7eb536a42383","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:33:12.741Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"1d37ce18-3b44-4cd0-9bc6-7eb536a42383","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:33:12.741Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"1d37ce18-3b44-4cd0-9bc6-7eb536a42383","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:33:12.741Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"1d37ce18-3b44-4cd0-9bc6-7eb536a42383"}
{"level":"info","ts":"2026-11-03T15:33:12.741Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"1d37ce18-3b44-4cd0-9bc6-7eb536a42383","RayCluster":"submitter-timeout-g4hph","State":""}
{"level":"info","ts":"2026-11-03T15:33:12.741Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"1d37ce18-3b44-4cd0-9bc6-7eb536a42383","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:33:09Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:33:09Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:33:15.742Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"3e4708d7-8753-4e4f-90c0-1236227bde85","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:33:15.742Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"3e4708d7-8753-4e4f-90c0-1236227bde85","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:33:15.742Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"3e4708d7-8753-4e4f-90c0-1236227bde85","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:33:15.742Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"3e4708d7-8753-4e4f-90c0-1236227bde85"}
{"level":"info","ts":"2026-11-03T15:33:15.742Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"3e4708d7-8753-4e4f-90c0-1236227bde85","RayCluster":"submitter-timeout-g4hph","State":""}
{"level":"info","ts":"2026-11-03T15:33:15.742Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"3e4708d7-8753-4e4f-90c0-1236227bde85","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:33:09Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:33:09Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:33:18.743Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"41df70c0-cf3b-4f01-b46f-def48be58849","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:33:18.743Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"41df70c0-cf3b-4f01-b46f-def48be58849","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:33:18.743Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"41df70c0-cf3b-4f01-b46f-def48be58849","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:33:18.743Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"41df70c0-cf3b-4f01-b46f-def48be58849"}
{"level":"info","ts":"2026-11-03T15:33:18.743Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"41df70c0-cf3b-4f01-b46f-def48be58849","RayCluster":"submitter-timeout-g4hph","State":""}
{"level":"info","ts":"2026-11-03T15:33:18.743Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"41df70c0-cf3b-4f01-b46f-def48be58849","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:33:09Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:33:09Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:33:19.405Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"d264ef3e-3fa8-4850-8376-bd385300dcba"}
{"level":"info","ts":"2026-11-03T15:33:19.405Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"d264ef3e-3fa8-4850-8376-bd385300dcba"}
{"level":"info","ts":"2026-11-03T15:33:19.405Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"d264ef3e-3fa8-4850-8376-bd385300dcba","1 head service found":"submitter-timeout-g4hph-head-svc"}
{"level":"info","ts":"2026-11-03T15:33:19.405Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"d264ef3e-3fa8-4850-8376-bd385300dcba","Found 1 head Pod":"submitter-timeout-g4hph-head-zjgxv","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:33:19.405Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"d264ef3e-3fa8-4850-8376-bd385300dcba","head Pod":"submitter-timeout-g4hph-head-zjgxv","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod submitter-timeout-g4hph-head-zjgxv. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:33:19.405Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"d264ef3e-3fa8-4850-8376-bd385300dcba","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:33:19.405Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"d264ef3e-3fa8-4850-8376-bd385300dcba","worker Pod":"submitter-timeout-g4hph-small-group-worker-jsjx7","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod submitter-timeout-g4hph-small-group-worker-jsjx7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:33:19.405Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"d264ef3e-3fa8-4850-8376-bd385300dcba","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:33:19.405Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"d264ef3e-3fa8-4850-8376-bd385300dcba","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:33:19.405Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"d264ef3e-3fa8-4850-8376-bd385300dcba","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:33:19.405Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"d264ef3e-3fa8-4850-8376-bd385300dcba","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:33:19.405Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"d264ef3e-3fa8-4850-8376-bd385300dcba","podName":"submitter-timeout-g4hph-head-zjgxv","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:31:57Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:33:19.406Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"d264ef3e-3fa8-4850-8376-bd385300dcba","podName":"submitter-timeout-g4hph-head-zjgxv","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:31:57Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:33:19.406Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"d264ef3e-3fa8-4850-8376-bd385300dcba","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:33:19.406Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"d264ef3e-3fa8-4850-8376-bd385300dcba","seconds":300}
{"level":"info","ts":"2026-11-03T15:33:20.414Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"03cf5412-1310-4b7b-a3f3-c8a92121824d"}
{"level":"info","ts":"2026-11-03T15:33:20.414Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"03cf5412-1310-4b7b-a3f3-c8a92121824d"}
{"level":"info","ts":"2026-11-03T15:33:20.414Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"03cf5412-1310-4b7b-a3f3-c8a92121824d","1 head service found":"submitter-timeout-g4hph-head-svc"}
{"level":"info","ts":"2026-11-03T15:33:20.414Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"03cf5412-1310-4b7b-a3f3-c8a92121824d","Found 1 head Pod":"submitter-timeout-g4hph-head-zjgxv","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:33:20.414Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"03cf5412-1310-4b7b-a3f3-c8a92121824d","head Pod":"submitter-timeout-g4hph-head-zjgxv","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod submitter-timeout-g4hph-head-zjgxv. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:33:20.414Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"03cf5412-1310-4b7b-a3f3-c8a92121824d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:33:20.414Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"03cf5412-1310-4b7b-a3f3-c8a92121824d","worker Pod":"submitter-timeout-g4hph-small-group-worker-jsjx7","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod submitter-timeout-g4hph-small-group-worker-jsjx7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:33:20.414Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"03cf5412-1310-4b7b-a3f3-c8a92121824d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:33:20.414Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"03cf5412-1310-4b7b-a3f3-c8a92121824d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:33:20.414Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"03cf5412-1310-4b7b-a3f3-c8a92121824d","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:33:20.414Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"03cf5412-1310-4b7b-a3f3-c8a92121824d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:33:20.414Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"03cf5412-1310-4b7b-a3f3-c8a92121824d","podName":"submitter-timeout-g4hph-head-zjgxv","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:31:57Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:33:20.414Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"03cf5412-1310-4b7b-a3f3-c8a92121824d","podName":"submitter-timeout-g4hph-head-zjgxv","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:31:57Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:33:20.415Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"03cf5412-1310-4b7b-a3f3-c8a92121824d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:33:20.415Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"03cf5412-1310-4b7b-a3f3-c8a92121824d","seconds":300}
{"level":"info","ts":"2026-11-03T15:33:20.650Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"ca67eec4-5f44-4e7a-847e-d475174152f7"}
{"level":"info","ts":"2026-11-03T15:33:20.650Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"ca67eec4-5f44-4e7a-847e-d475174152f7"}
{"level":"info","ts":"2026-11-03T15:33:20.650Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"ca67eec4-5f44-4e7a-847e-d475174152f7","1 head service found":"submitter-timeout-g4hph-head-svc"}
{"level":"info","ts":"2026-11-03T15:33:20.651Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"ca67eec4-5f44-4e7a-847e-d475174152f7","Found 1 head Pod":"submitter-timeout-g4hph-head-zjgxv","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:33:20.651Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"ca67eec4-5f44-4e7a-847e-d475174152f7","head Pod":"submitter-timeout-g4hph-head-zjgxv","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod submitter-timeout-g4hph-head-zjgxv. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:33:20.651Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"ca67eec4-5f44-4e7a-847e-d475174152f7","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:33:20.651Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"ca67eec4-5f44-4e7a-847e-d475174152f7","worker Pod":"submitter-timeout-g4hph-small-group-worker-jsjx7","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod submitter-timeout-g4hph-small-group-worker-jsjx7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:33:20.651Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"ca67eec4-5f44-4e7a-847e-d475174152f7","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:33:20.651Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"ca67eec4-5f44-4e7a-847e-d475174152f7","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:33:20.651Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"ca67eec4-5f44-4e7a-847e-d475174152f7","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:33:20.651Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"ca67eec4-5f44-4e7a-847e-d475174152f7","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:33:20.651Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"ca67eec4-5f44-4e7a-847e-d475174152f7","podName":"submitter-timeout-g4hph-small-group-worker-jsjx7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:33:20.651Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"ca67eec4-5f44-4e7a-847e-d475174152f7","podName":"submitter-timeout-g4hph-small-group-worker-jsjx7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:33:20.651Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"ca67eec4-5f44-4e7a-847e-d475174152f7","name":"submitter-timeout-g4hph","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:33:09Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:33:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:33:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:33:20.718Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"43f59883-99c1-4d7b-adc0-cd1c58affd3b","cluster":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:33:20.718Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"41baccc1-503f-4e8f-9b6f-e32c176112bd","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:33:20.718Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"41baccc1-503f-4e8f-9b6f-e32c176112bd","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:33:20.718Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"41baccc1-503f-4e8f-9b6f-e32c176112bd","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:33:20.718Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"41baccc1-503f-4e8f-9b6f-e32c176112bd"}
{"level":"info","ts":"2026-11-03T15:33:20.718Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"41baccc1-503f-4e8f-9b6f-e32c176112bd","RayCluster":"submitter-timeout-g4hph","State":""}
{"level":"info","ts":"2026-11-03T15:33:20.718Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"41baccc1-503f-4e8f-9b6f-e32c176112bd","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:33:09Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:33:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:33:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:33:20.718Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"41baccc1-503f-4e8f-9b6f-e32c176112bd","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:33:20.781Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"b305d3d7-a752-4a7a-999e-bf74684cab4a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:33:20.781Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"b305d3d7-a752-4a7a-999e-bf74684cab4a","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:33:20.781Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"b305d3d7-a752-4a7a-999e-bf74684cab4a","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:33:20.782Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"b305d3d7-a752-4a7a-999e-bf74684cab4a"}
{"level":"info","ts":"2026-11-03T15:33:20.782Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"b305d3d7-a752-4a7a-999e-bf74684cab4a","RayCluster":"submitter-timeout-g4hph","State":""}
{"level":"info","ts":"2026-11-03T15:33:20.782Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"b305d3d7-a752-4a7a-999e-bf74684cab4a","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:33:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:33:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:33:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:33:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:33:22.718Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"e82c20f4-e8d4-4199-b5a2-41164d3510d4"}
{"level":"info","ts":"2026-11-03T15:33:22.718Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"e82c20f4-e8d4-4199-b5a2-41164d3510d4"}
{"level":"info","ts":"2026-11-03T15:33:22.718Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"e82c20f4-e8d4-4199-b5a2-41164d3510d4","1 head service found":"submitter-timeout-g4hph-head-svc"}
{"level":"info","ts":"2026-11-03T15:33:22.719Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"e82c20f4-e8d4-4199-b5a2-41164d3510d4","Found 1 head Pod":"submitter-timeout-g4hph-head-zjgxv","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:33:22.719Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"e82c20f4-e8d4-4199-b5a2-41164d3510d4","head Pod":"submitter-timeout-g4hph-head-zjgxv","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod submitter-timeout-g4hph-head-zjgxv. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:33:22.719Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"e82c20f4-e8d4-4199-b5a2-41164d3510d4","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:33:22.719Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"e82c20f4-e8d4-4199-b5a2-41164d3510d4","worker Pod":"submitter-timeout-g4hph-small-group-worker-jsjx7","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod submitter-timeout-g4hph-small-group-worker-jsjx7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:33:22.719Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"e82c20f4-e8d4-4199-b5a2-41164d3510d4","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:33:22.719Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"e82c20f4-e8d4-4199-b5a2-41164d3510d4","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:33:22.719Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"e82c20f4-e8d4-4199-b5a2-41164d3510d4","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:33:22.719Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"e82c20f4-e8d4-4199-b5a2-41164d3510d4","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:33:22.719Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"e82c20f4-e8d4-4199-b5a2-41164d3510d4","podName":"submitter-timeout-g4hph-small-group-worker-jsjx7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:33:22.719Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"e82c20f4-e8d4-4199-b5a2-41164d3510d4","podName":"submitter-timeout-g4hph-small-group-worker-jsjx7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:33:22.719Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"e82c20f4-e8d4-4199-b5a2-41164d3510d4","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:33:22.719Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"e82c20f4-e8d4-4199-b5a2-41164d3510d4","seconds":300}
{"level":"info","ts":"2026-11-03T15:33:23.782Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"1d5a9852-a2c6-417c-ab40-83ee0c7fda90","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:33:23.782Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"1d5a9852-a2c6-417c-ab40-83ee0c7fda90","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:33:23.782Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"1d5a9852-a2c6-417c-ab40-83ee0c7fda90","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:33:23.783Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"1d5a9852-a2c6-417c-ab40-83ee0c7fda90"}
{"level":"info","ts":"2026-11-03T15:33:23.783Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"1d5a9852-a2c6-417c-ab40-83ee0c7fda90","RayCluster":"submitter-timeout-g4hph","State":""}
{"level":"info","ts":"2026-11-03T15:33:23.783Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"1d5a9852-a2c6-417c-ab40-83ee0c7fda90","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:33:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:33:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:33:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:33:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:33:26.784Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"60b6e1c2-247a-476e-9bc0-55fb44fd93ff","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:33:26.784Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"60b6e1c2-247a-476e-9bc0-55fb44fd93ff","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:33:26.784Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"60b6e1c2-247a-476e-9bc0-55fb44fd93ff","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:33:26.784Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"60b6e1c2-247a-476e-9bc0-55fb44fd93ff"}
{"level":"info","ts":"2026-11-03T15:33:26.784Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"60b6e1c2-247a-476e-9bc0-55fb44fd93ff","RayCluster":"submitter-timeout-g4hph","State":""}
{"level":"info","ts":"2026-11-03T15:33:26.784Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"60b6e1c2-247a-476e-9bc0-55fb44fd93ff","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:33:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:33:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:33:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:33:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:33:29.451Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"e8bb26ff-7574-49e1-b253-7fa0562a63d9"}
{"level":"info","ts":"2026-11-03T15:33:29.451Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"e8bb26ff-7574-49e1-b253-7fa0562a63d9"}
{"level":"info","ts":"2026-11-03T15:33:29.451Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"e8bb26ff-7574-49e1-b253-7fa0562a63d9","1 head service found":"submitter-timeout-g4hph-head-svc"}
{"level":"info","ts":"2026-11-03T15:33:29.452Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"e8bb26ff-7574-49e1-b253-7fa0562a63d9","Found 1 head Pod":"submitter-timeout-g4hph-head-zjgxv","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:33:29.452Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"e8bb26ff-7574-49e1-b253-7fa0562a63d9","head Pod":"submitter-timeout-g4hph-head-zjgxv","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod submitter-timeout-g4hph-head-zjgxv. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:33:29.452Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"e8bb26ff-7574-49e1-b253-7fa0562a63d9","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:33:29.452Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"e8bb26ff-7574-49e1-b253-7fa0562a63d9","worker Pod":"submitter-timeout-g4hph-small-group-worker-jsjx7","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod submitter-timeout-g4hph-small-group-worker-jsjx7. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:33:29.452Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"e8bb26ff-7574-49e1-b253-7fa0562a63d9","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:33:29.452Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"e8bb26ff-7574-49e1-b253-7fa0562a63d9","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:33:29.452Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"e8bb26ff-7574-49e1-b253-7fa0562a63d9","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:33:29.452Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"e8bb26ff-7574-49e1-b253-7fa0562a63d9","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:33:29.452Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"e8bb26ff-7574-49e1-b253-7fa0562a63d9","podName":"submitter-timeout-g4hph-small-group-worker-jsjx7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:33:29.452Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"e8bb26ff-7574-49e1-b253-7fa0562a63d9","podName":"submitter-timeout-g4hph-small-group-worker-jsjx7","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:33:29.452Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"e8bb26ff-7574-49e1-b253-7fa0562a63d9","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:33:29.452Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"e8bb26ff-7574-49e1-b253-7fa0562a63d9","seconds":300}
{"level":"info","ts":"2026-11-03T15:33:29.785Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"fd7899f4-0b78-4fd1-8c37-67e3f13033cb","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:33:29.785Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"fd7899f4-0b78-4fd1-8c37-67e3f13033cb","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:33:29.785Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"fd7899f4-0b78-4fd1-8c37-67e3f13033cb","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:33:29.785Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"fd7899f4-0b78-4fd1-8c37-67e3f13033cb"}
{"level":"info","ts":"2026-11-03T15:33:29.785Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"fd7899f4-0b78-4fd1-8c37-67e3f13033cb","RayCluster":"submitter-timeout-g4hph","State":""}
{"level":"info","ts":"2026-11-03T15:33:29.785Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"fd7899f4-0b78-4fd1-8c37-67e3f13033cb","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:33:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:33:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:33:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:33:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:33:30.511Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"fb962719-5be4-4b4d-9356-6451ebc61ed8"}
{"level":"info","ts":"2026-11-03T15:33:30.511Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"fb962719-5be4-4b4d-9356-6451ebc61ed8"}
{"level":"info","ts":"2026-11-03T15:33:30.511Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"fb962719-5be4-4b4d-9356-6451ebc61ed8","1 head service found":"submitter-timeout-g4hph-head-svc"}
{"level":"info","ts":"2026-11-03T15:33:30.512Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"fb962719-5be4-4b4d-9356-6451ebc61ed8","Found 1 head Pod":"submitter-timeout-g4hph-head-zjgxv","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:33:30.512Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"fb962719-5be4-4b4d-9356-6451ebc61ed8","head Pod":"submitter-timeout-g4hph-head-zjgxv","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod submitter-timeout-g4hph-head-zjgxv. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:33:30.512Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"fb962719-5be4-4b4d-9356-6451ebc61ed8","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:33:30.512Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"fb962719-5be4-4b4d-9356-6451ebc61ed8","worker Pod":"submitter-timeout-g4hph-small-group-worker-jsjx7","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod submitter-timeout-g4hph-small-group-worker-jsjx7. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:33:30.512Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"fb962719-5be4-4b4d-9356-6451ebc61ed8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:33:30.512Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"fb962719-5be4-4b4d-9356-6451ebc61ed8","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:33:30.512Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"fb962719-5be4-4b4d-9356-6451ebc61ed8","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:33:30.512Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"fb962719-5be4-4b4d-9356-6451ebc61ed8","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:33:30.512Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"fb962719-5be4-4b4d-9356-6451ebc61ed8","podName":"submitter-timeout-g4hph-small-group-worker-jsjx7","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:31:57Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:33:30.512Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"fb962719-5be4-4b4d-9356-6451ebc61ed8","podName":"submitter-timeout-g4hph-small-group-worker-jsjx7","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:31:57Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:33:30.512Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"fb962719-5be4-4b4d-9356-6451ebc61ed8","name":"submitter-timeout-g4hph","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:33:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:33:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:33:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:33:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:33:30.574Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"969ccd8f-6882-42ef-99be-d34dc3193580","cluster":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:33:30.574Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"db32e868-230a-4007-a19b-7ae87f44151a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:33:30.574Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"db32e868-230a-4007-a19b-7ae87f44151a","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:33:30.574Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"db32e868-230a-4007-a19b-7ae87f44151a","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:33:30.574Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"db32e868-230a-4007-a19b-7ae87f44151a"}
{"level":"info","ts":"2026-11-03T15:33:30.574Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"db32e868-230a-4007-a19b-7ae87f44151a","RayCluster":"submitter-timeout-g4hph","State":""}
{"level":"info","ts":"2026-11-03T15:33:30.574Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"db32e868-230a-4007-a19b-7ae87f44151a","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:33:20Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:33:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:33:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:33:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:33:30.575Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"db32e868-230a-4007-a19b-7ae87f44151a","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:33:30.659Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"b42a63a1-935a-4c0f-a950-f3d24ecd29d4","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:33:30.659Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"b42a63a1-935a-4c0f-a950-f3d24ecd29d4","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:33:30.659Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"b42a63a1-935a-4c0f-a950-f3d24ecd29d4","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:33:30.660Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"b42a63a1-935a-4c0f-a950-f3d24ecd29d4"}
{"level":"info","ts":"2026-11-03T15:33:30.660Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"b42a63a1-935a-4c0f-a950-f3d24ecd29d4","RayCluster":"submitter-timeout-g4hph","State":""}
{"level":"info","ts":"2026-11-03T15:33:30.660Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"b42a63a1-935a-4c0f-a950-f3d24ecd29d4","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:33:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:33:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:33:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:33:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:33:32.574Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"2794bcdf-797e-4b81-be4a-ddc5188cdd84"}
{"level":"info","ts":"2026-11-03T15:33:32.574Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"2794bcdf-797e-4b81-be4a-ddc5188cdd84"}
{"level":"info","ts":"2026-11-03T15:33:32.574Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"2794bcdf-797e-4b81-be4a-ddc5188cdd84","1 head service found":"submitter-timeout-g4hph-head-svc"}
{"level":"info","ts":"2026-11-03T15:33:32.574Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"2794bcdf-797e-4b81-be4a-ddc5188cdd84","Found 1 head Pod":"submitter-timeout-g4hph-head-zjgxv","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:33:32.574Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"2794bcdf-797e-4b81-be4a-ddc5188cdd84","head Pod":"submitter-timeout-g4hph-head-zjgxv","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod submitter-timeout-g4hph-head-zjgxv. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:33:32.574Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"2794bcdf-797e-4b81-be4a-ddc5188cdd84","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:33:32.574Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"2794bcdf-797e-4b81-be4a-ddc5188cdd84","worker Pod":"submitter-timeout-g4hph-small-group-worker-jsjx7","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod submitter-timeout-g4hph-small-group-worker-jsjx7. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:33:32.574Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"2794bcdf-797e-4b81-be4a-ddc5188cdd84","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:33:32.574Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"2794bcdf-797e-4b81-be4a-ddc5188cdd84","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:33:32.574Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"2794bcdf-797e-4b81-be4a-ddc5188cdd84","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:33:32.574Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"2794bcdf-797e-4b81-be4a-ddc5188cdd84","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:33:32.575Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"2794bcdf-797e-4b81-be4a-ddc5188cdd84","podName":"submitter-timeout-g4hph-small-group-worker-jsjx7","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:31:57Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:33:32.575Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"2794bcdf-797e-4b81-be4a-ddc5188cdd84","podName":"submitter-timeout-g4hph-small-group-worker-jsjx7","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:31:57Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:33:32.575Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"2794bcdf-797e-4b81-be4a-ddc5188cdd84","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:33:32.575Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"2794bcdf-797e-4b81-be4a-ddc5188cdd84","seconds":300}
{"level":"info","ts":"2026-11-03T15:33:33.660Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"0339a042-c7cd-4985-868e-b9442db20bca","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:33:33.660Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"0339a042-c7cd-4985-868e-b9442db20bca","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:33:33.660Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"0339a042-c7cd-4985-868e-b9442db20bca","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:33:33.661Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"0339a042-c7cd-4985-868e-b9442db20bca"}
{"level":"info","ts":"2026-11-03T15:33:33.661Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"0339a042-c7cd-4985-868e-b9442db20bca","RayCluster":"submitter-timeout-g4hph","State":""}
{"level":"info","ts":"2026-11-03T15:33:33.661Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"0339a042-c7cd-4985-868e-b9442db20bca","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:33:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:33:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:33:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:33:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:33:36.662Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"056a10ab-60b8-4f9d-a0c3-e877ad22f273","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:33:36.662Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"056a10ab-60b8-4f9d-a0c3-e877ad22f273","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:33:36.662Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"056a10ab-60b8-4f9d-a0c3-e877ad22f273","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:33:36.662Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"056a10ab-60b8-4f9d-a0c3-e877ad22f273"}
{"level":"info","ts":"2026-11-03T15:33:36.662Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"056a10ab-60b8-4f9d-a0c3-e877ad22f273","RayCluster":"submitter-timeout-g4hph","State":""}
{"level":"info","ts":"2026-11-03T15:33:36.662Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"056a10ab-60b8-4f9d-a0c3-e877ad22f273","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:33:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:33:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:33:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:33:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:33:39.663Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"d60e0808-7f40-4aeb-82d8-e50c89044370","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:33:39.663Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"d60e0808-7f40-4aeb-82d8-e50c89044370","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:33:39.663Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"d60e0808-7f40-4aeb-82d8-e50c89044370","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:33:39.663Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"d60e0808-7f40-4aeb-82d8-e50c89044370"}
{"level":"info","ts":"2026-11-03T15:33:39.663Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"d60e0808-7f40-4aeb-82d8-e50c89044370","RayCluster":"submitter-timeout-g4hph","State":""}
{"level":"info","ts":"2026-11-03T15:33:39.663Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"d60e0808-7f40-4aeb-82d8-e50c89044370","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:33:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:33:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:33:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:33:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:33:41.449Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"43c246bc-d0b2-4853-b98a-5076aeb09cfa","cluster":"long-running-hvml9"}
{"level":"info","ts":"2026-11-03T15:33:41.472Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"7f7082bb-d8f3-405e-bed3-92af9831b013"}
{"level":"info","ts":"2026-11-03T15:33:41.472Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"7f7082bb-d8f3-405e-bed3-92af9831b013"}
{"level":"info","ts":"2026-11-03T15:33:41.472Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"7f7082bb-d8f3-405e-bed3-92af9831b013","1 head service found":"submitter-timeout-g4hph-head-svc"}
{"level":"info","ts":"2026-11-03T15:33:41.473Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"7f7082bb-d8f3-405e-bed3-92af9831b013","Found 1 head Pod":"submitter-timeout-g4hph-head-zjgxv","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:33:41.473Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"7f7082bb-d8f3-405e-bed3-92af9831b013","head Pod":"submitter-timeout-g4hph-head-zjgxv","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod submitter-timeout-g4hph-head-zjgxv. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:33:41.473Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"7f7082bb-d8f3-405e-bed3-92af9831b013","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:33:41.473Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"7f7082bb-d8f3-405e-bed3-92af9831b013","worker Pod":"submitter-timeout-g4hph-small-group-worker-jsjx7","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod submitter-timeout-g4hph-small-group-worker-jsjx7. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:33:41.473Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"7f7082bb-d8f3-405e-bed3-92af9831b013","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:33:41.473Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"7f7082bb-d8f3-405e-bed3-92af9831b013","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:33:41.473Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"7f7082bb-d8f3-405e-bed3-92af9831b013","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:33:41.473Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"7f7082bb-d8f3-405e-bed3-92af9831b013","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:33:41.473Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"7f7082bb-d8f3-405e-bed3-92af9831b013","name":"submitter-timeout-g4hph","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:33:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:33:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:33:41Z","stateTransitionTimes":{"ready":"2026-11-03T15:33:41Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:33:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:33:41Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:33:41.534Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"4d6f0d64-c51e-4073-adc9-a86ae8e481cc","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:33:41.534Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"4d6f0d64-c51e-4073-adc9-a86ae8e481cc","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:33:41.534Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"4d6f0d64-c51e-4073-adc9-a86ae8e481cc","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:33:41.535Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"4d6f0d64-c51e-4073-adc9-a86ae8e481cc"}
{"level":"info","ts":"2026-11-03T15:33:41.599Z","logger":"controllers.RayJob","msg":"Created submitter Kubernetes Job for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"4d6f0d64-c51e-4073-adc9-a86ae8e481cc","Kubernetes Job":"submitter-timeout"}
{"level":"info","ts":"2026-11-03T15:33:41.599Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"4d6f0d64-c51e-4073-adc9-a86ae8e481cc","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:33:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:33:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","dashboardURL":"submitter-timeout-g4hph-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:33:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:33:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:33:41.599Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"4d6f0d64-c51e-4073-adc9-a86ae8e481cc","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-11-03T15:33:41.662Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"0bea2ceb-5e6d-4073-9ff0-99d1e84f4ce6","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:33:41.662Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"0bea2ceb-5e6d-4073-9ff0-99d1e84f4ce6","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:33:41.662Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"0bea2ceb-5e6d-4073-9ff0-99d1e84f4ce6","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:33:41.662Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"0bea2ceb-5e6d-4073-9ff0-99d1e84f4ce6"}
{"level":"info","ts":"2026-11-03T15:33:41.662Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job for RayJob already exists","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"0bea2ceb-5e6d-4073-9ff0-99d1e84f4ce6","Kubernetes Job":"submitter-timeout"}
{"level":"info","ts":"2026-11-03T15:33:41.662Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"0bea2ceb-5e6d-4073-9ff0-99d1e84f4ce6","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:33:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:33:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","dashboardURL":"submitter-timeout-g4hph-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:33:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:33:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:33:41.662Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"0bea2ceb-5e6d-4073-9ff0-99d1e84f4ce6","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-11-03T15:33:41.718Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"0bea2ceb-5e6d-4073-9ff0-99d1e84f4ce6","error":"Operation cannot be fulfilled on rayjobs.ray.io \"submitter-timeout\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-11-03T15:33:41.718Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"0bea2ceb-5e6d-4073-9ff0-99d1e84f4ce6"}
{"level":"error","ts":"2026-11-03T15:33:41.718Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"0bea2ceb-5e6d-4073-9ff0-99d1e84f4ce6","error":"Operation cannot be fulfilled on rayjobs.ray.io \"submitter-timeout\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:33:41.718Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"4a8f8a74-9213-4827-96d0-22b29858ebe5","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:33:41.718Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"4a8f8a74-9213-4827-96d0-22b29858ebe5","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:33:41.718Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"4a8f8a74-9213-4827-96d0-22b29858ebe5","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:33:41.718Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"4a8f8a74-9213-4827-96d0-22b29858ebe5"}
{"level":"info","ts":"2026-11-03T15:33:41.718Z","logger":"controllers.RayJob","msg":"Create a Kubernetes Service","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"4a8f8a74-9213-4827-96d0-22b29858ebe5","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:33:41.783Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"cc9e774f-4c9a-4fd9-ba70-8d8b78770480","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:33:41.783Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"cc9e774f-4c9a-4fd9-ba70-8d8b78770480","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:33:41.784Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"cc9e774f-4c9a-4fd9-ba70-8d8b78770480","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:33:41.784Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"cc9e774f-4c9a-4fd9-ba70-8d8b78770480"}
{"level":"info","ts":"2026-11-03T15:33:41.784Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"cc9e774f-4c9a-4fd9-ba70-8d8b78770480","rayCluster":"submitter-timeout-g4hph","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:33:42.316Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"b09ecc2c-b160-45b9-b24d-868ac7b50c2b","cluster":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:33:43.535Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"ec471bd3-3b33-4bfd-b929-e7b595ae93da"}
{"level":"info","ts":"2026-11-03T15:33:43.535Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"ec471bd3-3b33-4bfd-b929-e7b595ae93da"}
{"level":"info","ts":"2026-11-03T15:33:43.535Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"ec471bd3-3b33-4bfd-b929-e7b595ae93da","1 head service found":"submitter-timeout-g4hph-head-svc"}
{"level":"info","ts":"2026-11-03T15:33:43.535Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"ec471bd3-3b33-4bfd-b929-e7b595ae93da","Found 1 head Pod":"submitter-timeout-g4hph-head-zjgxv","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:33:43.535Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"ec471bd3-3b33-4bfd-b929-e7b595ae93da","head Pod":"submitter-timeout-g4hph-head-zjgxv","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod submitter-timeout-g4hph-head-zjgxv. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:33:43.535Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"ec471bd3-3b33-4bfd-b929-e7b595ae93da","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:33:43.535Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"ec471bd3-3b33-4bfd-b929-e7b595ae93da","worker Pod":"submitter-timeout-g4hph-small-group-worker-jsjx7","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod submitter-timeout-g4hph-small-group-worker-jsjx7. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:33:43.535Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"ec471bd3-3b33-4bfd-b929-e7b595ae93da","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:33:43.535Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"ec471bd3-3b33-4bfd-b929-e7b595ae93da","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:33:43.535Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"ec471bd3-3b33-4bfd-b929-e7b595ae93da","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:33:43.535Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"ec471bd3-3b33-4bfd-b929-e7b595ae93da","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:33:43.535Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"ec471bd3-3b33-4bfd-b929-e7b595ae93da","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:33:43.535Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"ec471bd3-3b33-4bfd-b929-e7b595ae93da","seconds":300}
{"level":"info","ts":"2026-11-03T15:33:44.473Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"7e670a3c-3530-48f2-9940-758113e3844d","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:33:44.473Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"7e670a3c-3530-48f2-9940-758113e3844d","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:33:44.473Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"7e670a3c-3530-48f2-9940-758113e3844d","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:33:44.473Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"7e670a3c-3530-48f2-9940-758113e3844d"}
{"level":"info","ts":"2026-11-03T15:33:44.473Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"7e670a3c-3530-48f2-9940-758113e3844d","rayCluster":"submitter-timeout-g4hph","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:33:47.479Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"a35a7a1d-48cc-4819-b950-2eed21e5e6c3","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:33:47.479Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"a35a7a1d-48cc-4819-b950-2eed21e5e6c3","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:33:47.479Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"a35a7a1d-48cc-4819-b950-2eed21e5e6c3","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:33:47.479Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"a35a7a1d-48cc-4819-b950-2eed21e5e6c3"}
{"level":"info","ts":"2026-11-03T15:33:47.479Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"a35a7a1d-48cc-4819-b950-2eed21e5e6c3","rayCluster":"submitter-timeout-g4hph","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:33:50.486Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"b2be8f7c-1c4c-4056-b490-43fb673e11e6","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:33:50.486Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"b2be8f7c-1c4c-4056-b490-43fb673e11e6","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:33:50.486Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"b2be8f7c-1c4c-4056-b490-43fb673e11e6","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:33:50.486Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"b2be8f7c-1c4c-4056-b490-43fb673e11e6"}
{"level":"info","ts":"2026-11-03T15:33:50.486Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"b2be8f7c-1c4c-4056-b490-43fb673e11e6","rayCluster":"submitter-timeout-g4hph","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:33:50.507Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"b2be8f7c-1c4c-4056-b490-43fb673e11e6","oldRayJobStatus":{"rayJobInfo":{},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","dashboardURL":"submitter-timeout-g4hph-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:33:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:33:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:33:50Z"},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","dashboardURL":"submitter-timeout-g4hph-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:33:41Z","stateTransitionTimes":{"ready":"2026-11-03T15:33:41Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:33:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:33:41Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:33:50.507Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"b2be8f7c-1c4c-4056-b490-43fb673e11e6","old JobStatus":"","new JobStatus":"PENDING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-11-03T15:33:50.604Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"363be7bc-7c32-469b-8b64-c3daf503a742","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:33:50.605Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"363be7bc-7c32-469b-8b64-c3daf503a742","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:33:50.605Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"363be7bc-7c32-469b-8b64-c3daf503a742","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:33:50.605Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"363be7bc-7c32-469b-8b64-c3daf503a742"}
{"level":"info","ts":"2026-11-03T15:33:50.605Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"363be7bc-7c32-469b-8b64-c3daf503a742","rayCluster":"submitter-timeout-g4hph","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:33:50.608Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"363be7bc-7c32-469b-8b64-c3daf503a742","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:33:50Z"},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","dashboardURL":"submitter-timeout-g4hph-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:33:41Z","stateTransitionTimes":{"ready":"2026-11-03T15:33:41Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:33:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:33:41Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:33:50Z"},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","dashboardURL":"submitter-timeout-g4hph-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:33:41Z","stateTransitionTimes":{"ready":"2026-11-03T15:33:41Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:33:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:33:41Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:33:53.609Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"75d9a8f0-dc46-45dc-922c-40b330d63a3a","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:33:53.609Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"75d9a8f0-dc46-45dc-922c-40b330d63a3a","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:33:53.609Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"75d9a8f0-dc46-45dc-922c-40b330d63a3a","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:33:53.609Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"75d9a8f0-dc46-45dc-922c-40b330d63a3a"}
{"level":"info","ts":"2026-11-03T15:33:53.609Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"75d9a8f0-dc46-45dc-922c-40b330d63a3a","rayCluster":"submitter-timeout-g4hph","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:33:53.612Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"75d9a8f0-dc46-45dc-922c-40b330d63a3a","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:33:50Z"},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","dashboardURL":"submitter-timeout-g4hph-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:33:41Z","stateTransitionTimes":{"ready":"2026-11-03T15:33:41Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:33:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:33:41Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:33:50Z"},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","dashboardURL":"submitter-timeout-g4hph-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:33:41Z","stateTransitionTimes":{"ready":"2026-11-03T15:33:41Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:33:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:33:41Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:33:53.612Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"75d9a8f0-dc46-45dc-922c-40b330d63a3a","old JobStatus":"PENDING","new JobStatus":"RUNNING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-11-03T15:33:53.710Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"8afd8898-faa2-4f77-802f-19bb74e0dabc","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:33:53.710Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"8afd8898-faa2-4f77-802f-19bb74e0dabc","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:33:53.710Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"8afd8898-faa2-4f77-802f-19bb74e0dabc","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:33:53.710Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"8afd8898-faa2-4f77-802f-19bb74e0dabc"}
{"level":"info","ts":"2026-11-03T15:33:53.710Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"8afd8898-faa2-4f77-802f-19bb74e0dabc","rayCluster":"submitter-timeout-g4hph","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:33:53.713Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"8afd8898-faa2-4f77-802f-19bb74e0dabc","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:33:50Z"},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","dashboardURL":"submitter-timeout-g4hph-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:33:41Z","stateTransitionTimes":{"ready":"2026-11-03T15:33:41Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:33:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:33:41Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:33:50Z"},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","dashboardURL":"submitter-timeout-g4hph-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:33:41Z","stateTransitionTimes":{"ready":"2026-11-03T15:33:41Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:33:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:33:41Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:33:53.944Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"5dad222c-2297-4b75-9fe8-d56bfc6bc9f6","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:33:53.944Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"5dad222c-2297-4b75-9fe8-d56bfc6bc9f6","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:33:53.944Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"5dad222c-2297-4b75-9fe8-d56bfc6bc9f6","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:33:53.944Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"5dad222c-2297-4b75-9fe8-d56bfc6bc9f6"}
{"level":"info","ts":"2026-11-03T15:33:53.944Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"5dad222c-2297-4b75-9fe8-d56bfc6bc9f6","rayCluster":"submitter-timeout-g4hph","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:33:53.946Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"5dad222c-2297-4b75-9fe8-d56bfc6bc9f6","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:33:50Z"},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","dashboardURL":"submitter-timeout-g4hph-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:33:41Z","stateTransitionTimes":{"ready":"2026-11-03T15:33:41Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:33:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:33:41Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:33:50Z"},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","dashboardURL":"submitter-timeout-g4hph-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:33:41Z","stateTransitionTimes":{"ready":"2026-11-03T15:33:41Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:33:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:33:41Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:33:56.948Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"ad4515eb-aad4-437e-a0fa-a30a86bfc7b3","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:33:56.948Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"ad4515eb-aad4-437e-a0fa-a30a86bfc7b3","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:33:56.948Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"ad4515eb-aad4-437e-a0fa-a30a86bfc7b3","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:33:56.948Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"ad4515eb-aad4-437e-a0fa-a30a86bfc7b3"}
{"level":"info","ts":"2026-11-03T15:33:56.948Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"ad4515eb-aad4-437e-a0fa-a30a86bfc7b3","rayCluster":"submitter-timeout-g4hph","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:33:56.950Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"ad4515eb-aad4-437e-a0fa-a30a86bfc7b3","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:33:50Z"},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","dashboardURL":"submitter-timeout-g4hph-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:33:41Z","stateTransitionTimes":{"ready":"2026-11-03T15:33:41Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:33:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:33:41Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:33:50Z"},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","dashboardURL":"submitter-timeout-g4hph-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:33:41Z","stateTransitionTimes":{"ready":"2026-11-03T15:33:41Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:33:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:33:41Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:33:59.951Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"a9f16dc9-9bb9-4fab-bb60-88f44acf8760","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:33:59.951Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"a9f16dc9-9bb9-4fab-bb60-88f44acf8760","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:33:59.951Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"a9f16dc9-9bb9-4fab-bb60-88f44acf8760","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:33:59.952Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"a9f16dc9-9bb9-4fab-bb60-88f44acf8760"}
{"level":"info","ts":"2026-11-03T15:33:59.952Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"a9f16dc9-9bb9-4fab-bb60-88f44acf8760","rayCluster":"submitter-timeout-g4hph","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:33:59.954Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"a9f16dc9-9bb9-4fab-bb60-88f44acf8760","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:33:50Z"},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","dashboardURL":"submitter-timeout-g4hph-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:33:41Z","stateTransitionTimes":{"ready":"2026-11-03T15:33:41Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:33:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:33:41Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:33:50Z"},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","dashboardURL":"submitter-timeout-g4hph-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:33:41Z","stateTransitionTimes":{"ready":"2026-11-03T15:33:41Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:33:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:33:41Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:34:02.955Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"814f88c3-c499-4631-92fb-1acb09676295","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:34:02.955Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"814f88c3-c499-4631-92fb-1acb09676295","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:34:02.955Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"814f88c3-c499-4631-92fb-1acb09676295","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:34:02.955Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"814f88c3-c499-4631-92fb-1acb09676295"}
{"level":"info","ts":"2026-11-03T15:34:02.955Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"814f88c3-c499-4631-92fb-1acb09676295","rayCluster":"submitter-timeout-g4hph","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:34:02.958Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"814f88c3-c499-4631-92fb-1acb09676295","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:33:50Z"},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","dashboardURL":"submitter-timeout-g4hph-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:33:41Z","stateTransitionTimes":{"ready":"2026-11-03T15:33:41Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:33:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:33:41Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:33:50Z"},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","dashboardURL":"submitter-timeout-g4hph-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:33:41Z","stateTransitionTimes":{"ready":"2026-11-03T15:33:41Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:33:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:33:41Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:34:05.959Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"d00328ea-ecb5-4b31-979d-ca26dc2a56f7","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:34:05.959Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"d00328ea-ecb5-4b31-979d-ca26dc2a56f7","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:34:05.959Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"d00328ea-ecb5-4b31-979d-ca26dc2a56f7","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:34:05.959Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"d00328ea-ecb5-4b31-979d-ca26dc2a56f7"}
{"level":"info","ts":"2026-11-03T15:34:05.959Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"d00328ea-ecb5-4b31-979d-ca26dc2a56f7","rayCluster":"submitter-timeout-g4hph","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:34:05.962Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"d00328ea-ecb5-4b31-979d-ca26dc2a56f7","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:33:50Z"},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","dashboardURL":"submitter-timeout-g4hph-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:33:41Z","stateTransitionTimes":{"ready":"2026-11-03T15:33:41Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:33:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:33:41Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:33:50Z"},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","dashboardURL":"submitter-timeout-g4hph-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:33:41Z","stateTransitionTimes":{"ready":"2026-11-03T15:33:41Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:33:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:33:41Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:34:08.963Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"871938d1-f047-4656-ab8b-4e745d6ee519","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:34:08.963Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"871938d1-f047-4656-ab8b-4e745d6ee519","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:34:08.963Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"871938d1-f047-4656-ab8b-4e745d6ee519","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:34:08.963Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"871938d1-f047-4656-ab8b-4e745d6ee519"}
{"level":"info","ts":"2026-11-03T15:34:08.963Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"871938d1-f047-4656-ab8b-4e745d6ee519","rayCluster":"submitter-timeout-g4hph","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:34:08.966Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"871938d1-f047-4656-ab8b-4e745d6ee519","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:33:50Z"},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","dashboardURL":"submitter-timeout-g4hph-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:33:41Z","stateTransitionTimes":{"ready":"2026-11-03T15:33:41Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:33:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:33:41Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:33:50Z"},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","dashboardURL":"submitter-timeout-g4hph-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:33:41Z","stateTransitionTimes":{"ready":"2026-11-03T15:33:41Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:33:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:33:41Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:34:11.967Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"af6bf8c1-d2f0-40ec-ab69-8a70cffadbfb","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:34:11.967Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"af6bf8c1-d2f0-40ec-ab69-8a70cffadbfb","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:34:11.967Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"af6bf8c1-d2f0-40ec-ab69-8a70cffadbfb","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:34:11.967Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"af6bf8c1-d2f0-40ec-ab69-8a70cffadbfb"}
{"level":"info","ts":"2026-11-03T15:34:11.967Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"af6bf8c1-d2f0-40ec-ab69-8a70cffadbfb","rayCluster":"submitter-timeout-g4hph","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:34:11.977Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"af6bf8c1-d2f0-40ec-ab69-8a70cffadbfb","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:33:50Z"},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","dashboardURL":"submitter-timeout-g4hph-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:33:41Z","stateTransitionTimes":{"ready":"2026-11-03T15:33:41Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:33:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:33:41Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:33:50Z"},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","dashboardURL":"submitter-timeout-g4hph-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:33:41Z","stateTransitionTimes":{"ready":"2026-11-03T15:33:41Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:33:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:33:41Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:34:14.978Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"86eb9171-488a-4a15-94f7-f5e91a726dd8","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:34:14.978Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"86eb9171-488a-4a15-94f7-f5e91a726dd8","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:34:14.978Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"86eb9171-488a-4a15-94f7-f5e91a726dd8","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:34:14.978Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"86eb9171-488a-4a15-94f7-f5e91a726dd8"}
{"level":"info","ts":"2026-11-03T15:34:14.978Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"86eb9171-488a-4a15-94f7-f5e91a726dd8","rayCluster":"submitter-timeout-g4hph","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:34:14.981Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"86eb9171-488a-4a15-94f7-f5e91a726dd8","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:33:50Z"},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","dashboardURL":"submitter-timeout-g4hph-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:33:41Z","stateTransitionTimes":{"ready":"2026-11-03T15:33:41Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:33:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:33:41Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:33:50Z"},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","dashboardURL":"submitter-timeout-g4hph-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:33:41Z","stateTransitionTimes":{"ready":"2026-11-03T15:33:41Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:33:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:33:41Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:34:17.982Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"2e9c739c-dfcb-47dd-9143-2814b05e8660","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:34:17.982Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"2e9c739c-dfcb-47dd-9143-2814b05e8660","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:34:17.982Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"2e9c739c-dfcb-47dd-9143-2814b05e8660","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:34:17.982Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"2e9c739c-dfcb-47dd-9143-2814b05e8660"}
{"level":"info","ts":"2026-11-03T15:34:17.982Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"2e9c739c-dfcb-47dd-9143-2814b05e8660","rayCluster":"submitter-timeout-g4hph","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:34:17.985Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"2e9c739c-dfcb-47dd-9143-2814b05e8660","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:33:50Z"},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","dashboardURL":"submitter-timeout-g4hph-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:33:41Z","stateTransitionTimes":{"ready":"2026-11-03T15:33:41Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:33:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:33:41Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:33:50Z"},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","dashboardURL":"submitter-timeout-g4hph-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:33:41Z","stateTransitionTimes":{"ready":"2026-11-03T15:33:41Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:33:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:33:41Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:34:20.986Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"e9c5fe02-afda-4598-b839-cbdcce4d3c32","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:34:20.986Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"e9c5fe02-afda-4598-b839-cbdcce4d3c32","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:34:20.986Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"e9c5fe02-afda-4598-b839-cbdcce4d3c32","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:34:20.986Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"e9c5fe02-afda-4598-b839-cbdcce4d3c32"}
{"level":"info","ts":"2026-11-03T15:34:20.987Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"e9c5fe02-afda-4598-b839-cbdcce4d3c32","rayCluster":"submitter-timeout-g4hph","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:34:20.989Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"e9c5fe02-afda-4598-b839-cbdcce4d3c32","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:33:50Z"},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","dashboardURL":"submitter-timeout-g4hph-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:33:41Z","stateTransitionTimes":{"ready":"2026-11-03T15:33:41Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:33:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:33:41Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:33:50Z"},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","dashboardURL":"submitter-timeout-g4hph-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:33:41Z","stateTransitionTimes":{"ready":"2026-11-03T15:33:41Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:33:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:33:41Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:34:23.990Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"5f02e6ae-c881-42ec-8985-734955e7abea","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:34:23.990Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"5f02e6ae-c881-42ec-8985-734955e7abea","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:34:23.990Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"5f02e6ae-c881-42ec-8985-734955e7abea","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:34:23.990Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"5f02e6ae-c881-42ec-8985-734955e7abea"}
{"level":"info","ts":"2026-11-03T15:34:23.990Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"5f02e6ae-c881-42ec-8985-734955e7abea","rayCluster":"submitter-timeout-g4hph","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:34:23.990Z","logger":"controllers.RayJob","msg":"The RayJob has passed the submitterFinishedTimeoutSeconds. Transition the status to terminal.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"5f02e6ae-c881-42ec-8985-734955e7abea","SubmitterFinishedTime":"2026-11-03T15:33:53.000Z","SubmitterFinishedTimeoutSeconds":"30s"}
{"level":"info","ts":"2026-11-03T15:34:23.990Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"5f02e6ae-c881-42ec-8985-734955e7abea","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:33:50Z"},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","dashboardURL":"submitter-timeout-g4hph-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:33:41Z","stateTransitionTimes":{"ready":"2026-11-03T15:33:41Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:33:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:33:41Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:33:50Z"},"jobId":"submitter-timeout-4hwht","rayClusterName":"submitter-timeout-g4hph","dashboardURL":"submitter-timeout-g4hph-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"FAILED","jobDeploymentStatus":"Failed","reason":"JobDeploymentStatusTransitionGracePeriodExceeded","message":"The RayJob submitter finished at 2026-11-03 15:33:53 but the ray job did not reach terminal state within 30s","startTime":"2026-11-03T15:31:53Z","succeeded":0,"failed":1,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:33:41Z","stateTransitionTimes":{"ready":"2026-11-03T15:33:41Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.50","serviceIP":"10.134.0.50","podName":"submitter-timeout-g4hph-head-zjgxv","serviceName":"submitter-timeout-g4hph-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:33:20Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:33:41Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:31:53Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:31:54Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:34:23.990Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"5f02e6ae-c881-42ec-8985-734955e7abea","old JobStatus":"RUNNING","new JobStatus":"FAILED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Failed"}
{"level":"info","ts":"2026-11-03T15:34:24.099Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"362c4dae-c249-4419-960e-fbe0be4a58a8","JobStatus":"FAILED","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:34:24.099Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"362c4dae-c249-4419-960e-fbe0be4a58a8","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Failed","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-11-03 15:34:23 +0000 UTC","Now":"2026-11-03T15:34:24.099Z","ShutdownTime":"2026-11-03T15:34:23.000Z"}
{"level":"info","ts":"2026-11-03T15:34:24.157Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob is deleted","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"362c4dae-c249-4419-960e-fbe0be4a58a8","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:34:24.157Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"362c4dae-c249-4419-960e-fbe0be4a58a8","isClusterDeleted":false}
{"level":"info","ts":"2026-11-03T15:34:24.157Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"362c4dae-c249-4419-960e-fbe0be4a58a8","deletionMechanism":"ShutdownAfterJobFinishes","RayCluster":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:34:24.158Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"reconcileID":"c1a180a5-ee3c-4631-9219-161399af5c0f"}
{"level":"info","ts":"2026-11-03T15:34:24.158Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"submitter-timeout-g4hph","reconcileID":"33bad780-18ce-416f-8306-ae079c289c12","namespacedName":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:34:24.158Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"7dfed0e0-f2d7-4803-a3f4-d9ea97bcf0b6","JobStatus":"FAILED","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:34:24.158Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"submitter-timeout-g4hph","reconcileID":"33bad780-18ce-416f-8306-ae079c289c12","cluster":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:34:24.158Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"submitter-timeout-g4hph","reconcileID":"33bad780-18ce-416f-8306-ae079c289c12","cluster":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:34:24.158Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"7dfed0e0-f2d7-4803-a3f4-d9ea97bcf0b6","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Failed","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-11-03 15:34:23 +0000 UTC","Now":"2026-11-03T15:34:24.158Z","ShutdownTime":"2026-11-03T15:34:23.000Z"}
{"level":"info","ts":"2026-11-03T15:34:24.158Z","logger":"controllers.RayJob","msg":"The deletion of the associated RayCluster for RayJob is ongoing.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"7dfed0e0-f2d7-4803-a3f4-d9ea97bcf0b6","RayCluster":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:34:24.158Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"7dfed0e0-f2d7-4803-a3f4-d9ea97bcf0b6","isClusterDeleted":false}
{"level":"info","ts":"2026-11-03T15:34:24.158Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"7dfed0e0-f2d7-4803-a3f4-d9ea97bcf0b6","deletionMechanism":"ShutdownAfterJobFinishes","RayCluster":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:34:24.158Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"submitter-timeout-g4hph","reconcileID":"33bad780-18ce-416f-8306-ae079c289c12","namespace":"test-ns-kbx4w","otherAuthClusters":1,"action":"will retain"}
{"level":"info","ts":"2026-11-03T15:34:24.158Z","logger":"authentication-controller","msg":"ReferenceGrant retained (other clusters with auth exist)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"submitter-timeout-g4hph","reconcileID":"33bad780-18ce-416f-8306-ae079c289c12","name":"kuberay-gateway-access","namespace":"test-ns-kbx4w","remainingClusters":1}
{"level":"info","ts":"2026-11-03T15:34:24.213Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"submitter-timeout-g4hph","reconcileID":"33bad780-18ce-416f-8306-ae079c289c12","name":"test-ns-kbx4w-submitter-timeout-g4hph","namespace":"opendatahub"}
{"level":"info","ts":"2026-11-03T15:34:24.265Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"submitter-timeout-g4hph","reconcileID":"33bad780-18ce-416f-8306-ae079c289c12","configMap":"kube-rbac-proxy-config-submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:34:24.317Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"submitter-timeout-g4hph","reconcileID":"33bad780-18ce-416f-8306-ae079c289c12","serviceAccount":"submitter-timeout-g4hph-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-11-03T15:34:24.500Z","logger":"raycluster-resource","msg":"default","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:34:24.500Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:34:24.588Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"cf239fd5-7cc1-449f-9e43-611a085cb99b","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:34:24.588Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"cf239fd5-7cc1-449f-9e43-611a085cb99b","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-11-03T15:34:24.648Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"cf239fd5-7cc1-449f-9e43-611a085cb99b"}
{"level":"info","ts":"2026-11-03T15:34:24.648Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"cf239fd5-7cc1-449f-9e43-611a085cb99b","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-11-03T15:34:24.648Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"cf239fd5-7cc1-449f-9e43-611a085cb99b","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:34:24.648Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"cf239fd5-7cc1-449f-9e43-611a085cb99b","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:34:24.681Z","logger":"raycluster-resource","msg":"validate update","name":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:34:24.708Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"93473eee-b29a-4832-b511-5ff57c2ed326","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:34:24.708Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"93473eee-b29a-4832-b511-5ff57c2ed326","name":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:34:24.708Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"93473eee-b29a-4832-b511-5ff57c2ed326","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:34:24.708Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"93473eee-b29a-4832-b511-5ff57c2ed326","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:34:24.736Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"submitter-timeout-g4hph","reconcileID":"33bad780-18ce-416f-8306-ae079c289c12","cluster":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:34:24.736Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"submitter-timeout-g4hph","reconcileID":"dfaafd84-d603-49ca-a1bc-3958ce55e1ab","namespacedName":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:34:24.736Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"submitter-timeout-g4hph","reconcileID":"dfaafd84-d603-49ca-a1bc-3958ce55e1ab","cluster":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:34:24.736Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"submitter-timeout-g4hph","reconcileID":"dfaafd84-d603-49ca-a1bc-3958ce55e1ab","cluster":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:34:24.736Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"submitter-timeout-g4hph","reconcileID":"dfaafd84-d603-49ca-a1bc-3958ce55e1ab","namespace":"test-ns-kbx4w","otherAuthClusters":1,"action":"will retain"}
{"level":"info","ts":"2026-11-03T15:34:24.736Z","logger":"authentication-controller","msg":"ReferenceGrant retained (other clusters with auth exist)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"submitter-timeout-g4hph","reconcileID":"dfaafd84-d603-49ca-a1bc-3958ce55e1ab","name":"kuberay-gateway-access","namespace":"test-ns-kbx4w","remainingClusters":1}
{"level":"info","ts":"2026-11-03T15:34:24.764Z","logger":"raycluster-resource","msg":"default","name":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:34:24.764Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}
{"level":"error","ts":"2026-11-03T15:34:24.805Z","logger":"authentication-controller","msg":"Failed to remove finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"submitter-timeout-g4hph","reconcileID":"dfaafd84-d603-49ca-a1bc-3958ce55e1ab","error":"rayclusters.ray.io \"submitter-timeout-g4hph\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleDeletion\n\t/workspace/controllers/ray/authentication_controller.go:195\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:34:24.805Z","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"submitter-timeout-g4hph","reconcileID":"dfaafd84-d603-49ca-a1bc-3958ce55e1ab"}
{"level":"error","ts":"2026-11-03T15:34:24.805Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"submitter-timeout-g4hph","reconcileID":"dfaafd84-d603-49ca-a1bc-3958ce55e1ab","error":"rayclusters.ray.io \"submitter-timeout-g4hph\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:34:24.805Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"submitter-timeout-g4hph","reconcileID":"258a8ff2-530a-4a8c-9176-903b7d9acad9","namespacedName":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:34:24.805Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"submitter-timeout-g4hph","reconcileID":"258a8ff2-530a-4a8c-9176-903b7d9acad9"}
{"level":"info","ts":"2026-11-03T15:34:24.805Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"submitter-timeout-g4hph","reconcileID":"258a8ff2-530a-4a8c-9176-903b7d9acad9","namespace":"test-ns-kbx4w","authClusters":1}
{"level":"info","ts":"2026-11-03T15:34:24.805Z","logger":"authentication-controller","msg":"ReferenceGrant not orphaned (clusters with auth exist)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"submitter-timeout-g4hph","reconcileID":"258a8ff2-530a-4a8c-9176-903b7d9acad9","namespace":"test-ns-kbx4w","authClusters":1}
{"level":"info","ts":"2026-11-03T15:34:24.828Z","logger":"raycluster-resource","msg":"validate create","name":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:34:24.884Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"93473eee-b29a-4832-b511-5ff57c2ed326","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:34:24.884Z","logger":"controllers.NetworkPolicy","msg":"Reconciling NetworkPolicies for RayCluster","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"6b51312e-9947-451f-b7e5-aeea0700ec76","cluster":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:34:24.884Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"93473eee-b29a-4832-b511-5ff57c2ed326"}
{"level":"info","ts":"2026-11-03T15:34:24.884Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"93473eee-b29a-4832-b511-5ff57c2ed326","RayCluster":"cluster-status-update-mgqjh","State":""}
{"level":"info","ts":"2026-11-03T15:34:24.884Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"93473eee-b29a-4832-b511-5ff57c2ed326","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:34:24.884Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"afd56d34-ceee-4c8e-8d35-19c0e951dfbc","JobStatus":"FAILED","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:34:24.884Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"afd56d34-ceee-4c8e-8d35-19c0e951dfbc","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Failed","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-11-03 15:34:23 +0000 UTC","Now":"2026-11-03T15:34:24.884Z","ShutdownTime":"2026-11-03T15:34:23.000Z"}
{"level":"info","ts":"2026-11-03T15:34:24.884Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"afd56d34-ceee-4c8e-8d35-19c0e951dfbc","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:34:24.884Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"afd56d34-ceee-4c8e-8d35-19c0e951dfbc","isClusterDeleted":true}
{"level":"info","ts":"2026-11-03T15:34:24.884Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"afd56d34-ceee-4c8e-8d35-19c0e951dfbc","deletionMechanism":"ShutdownAfterJobFinishes","RayCluster":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:34:24.884Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"388dff49-1681-4a8b-ab5b-4b75a6264be5","cluster":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:34:24.884Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"a4dc1560-d533-4513-941d-9f30d3c633be","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:34:24.884Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"a4dc1560-d533-4513-941d-9f30d3c633be","name":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:34:24.884Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"313f0cb7-112d-47e2-9923-2e7b5690d956"}
{"level":"info","ts":"2026-11-03T15:34:24.884Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"313f0cb7-112d-47e2-9923-2e7b5690d956"}
{"level":"info","ts":"2026-11-03T15:34:24.884Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"a4dc1560-d533-4513-941d-9f30d3c633be","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:34:24.885Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"cluster-status-update-mgqjh","reconcileID":"02fbdbf3-7eb3-4bef-a2db-7c0f4712917f","namespacedName":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:34:24.885Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"a4dc1560-d533-4513-941d-9f30d3c633be"}
{"level":"info","ts":"2026-11-03T15:34:24.885Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"cluster-status-update-mgqjh","reconcileID":"02fbdbf3-7eb3-4bef-a2db-7c0f4712917f","mode":"IntegratedOAuth","cluster":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:34:24.885Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"a4dc1560-d533-4513-941d-9f30d3c633be","RayCluster":"cluster-status-update-mgqjh","State":""}
{"level":"info","ts":"2026-11-03T15:34:24.885Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"a4dc1560-d533-4513-941d-9f30d3c633be","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:34:24.885Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"cluster-status-update-mgqjh","reconcileID":"02fbdbf3-7eb3-4bef-a2db-7c0f4712917f","cluster":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:34:24.885Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"cluster-status-update-mgqjh","reconcileID":"02fbdbf3-7eb3-4bef-a2db-7c0f4712917f","cluster":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:34:24.941Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"313f0cb7-112d-47e2-9923-2e7b5690d956","name":"cluster-status-update-mgqjh-head-svc"}
{"level":"info","ts":"2026-11-03T15:34:24.941Z","logger":"controllers.NetworkPolicy","msg":"Successfully created NetworkPolicy","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"6b51312e-9947-451f-b7e5-aeea0700ec76","name":"cluster-status-update-mgqjh-head"}
{"level":"info","ts":"2026-11-03T15:34:24.941Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"313f0cb7-112d-47e2-9923-2e7b5690d956"}
{"level":"info","ts":"2026-11-03T15:34:24.942Z","logger":"raycluster-resource","msg":"default","name":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:34:24.942Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:34:24.992Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"313f0cb7-112d-47e2-9923-2e7b5690d956","error":"certificate ray-head-cert-cluster-status-update-mgqjh not found: certificates.cert-manager.io \"ray-head-cert-cluster-status-update-mgqjh\" not found"}
{"level":"error","ts":"2026-11-03T15:34:24.992Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"313f0cb7-112d-47e2-9923-2e7b5690d956","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-cluster-status-update-mgqjh not found: certificates.cert-manager.io \"ray-head-cert-cluster-status-update-mgqjh\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:34:24.992Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"313f0cb7-112d-47e2-9923-2e7b5690d956","error":"mTLS secrets not ready: certificate ray-head-cert-cluster-status-update-mgqjh not found: certificates.cert-manager.io \"ray-head-cert-cluster-status-update-mgqjh\" not found"}
{"level":"info","ts":"2026-11-03T15:34:24.992Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"313f0cb7-112d-47e2-9923-2e7b5690d956","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"cluster-status-update-mgqjh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:34:24.992Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"313f0cb7-112d-47e2-9923-2e7b5690d956","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"cluster-status-update-mgqjh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:34:24.992Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"313f0cb7-112d-47e2-9923-2e7b5690d956","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"cluster-status-update-mgqjh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:34:24.992Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"313f0cb7-112d-47e2-9923-2e7b5690d956","name":"cluster-status-update-mgqjh","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:24Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:24Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:24Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:24Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:24Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:34:24.994Z","logger":"controllers.NetworkPolicy","msg":"Successfully created NetworkPolicy","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"6b51312e-9947-451f-b7e5-aeea0700ec76","name":"cluster-status-update-mgqjh-workers-small-group"}
{"level":"info","ts":"2026-11-03T15:34:24.994Z","logger":"controllers.NetworkPolicy","msg":"Successfully reconciled NetworkPolicies for RayCluster","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"6b51312e-9947-451f-b7e5-aeea0700ec76","cluster":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:34:24.994Z","logger":"controllers.NetworkPolicy","msg":"Reconciling NetworkPolicies for RayCluster","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"702036a1-77e4-4ab4-88cf-3b010411228b","cluster":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:34:25.002Z","logger":"raycluster-resource","msg":"validate update","name":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:34:25.062Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"388feab2-5f9d-4e14-a6ac-da8b8f7b84a2","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:34:25.062Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"388feab2-5f9d-4e14-a6ac-da8b8f7b84a2","name":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:34:25.062Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"388feab2-5f9d-4e14-a6ac-da8b8f7b84a2","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:34:25.062Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"388feab2-5f9d-4e14-a6ac-da8b8f7b84a2"}
{"level":"info","ts":"2026-11-03T15:34:25.062Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"388feab2-5f9d-4e14-a6ac-da8b8f7b84a2","RayCluster":"cluster-status-update-mgqjh","State":""}
{"level":"info","ts":"2026-11-03T15:34:25.062Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"388feab2-5f9d-4e14-a6ac-da8b8f7b84a2","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:34:25.066Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"313f0cb7-112d-47e2-9923-2e7b5690d956","name":"cluster-status-update-mgqjh","error":"Operation cannot be fulfilled on rayclusters.ray.io \"cluster-status-update-mgqjh\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-kbx4w","name":"cluster-status-update-mgqjh"}}
{"level":"info","ts":"2026-11-03T15:34:25.066Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"313f0cb7-112d-47e2-9923-2e7b5690d956"}
{"level":"error","ts":"2026-11-03T15:34:25.066Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"313f0cb7-112d-47e2-9923-2e7b5690d956","error":"mTLS secrets not ready: certificate ray-head-cert-cluster-status-update-mgqjh not found: certificates.cert-manager.io \"ray-head-cert-cluster-status-update-mgqjh\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:34:25.066Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"1f647f19-9640-47d3-8a54-9dd50efdbcc6"}
{"level":"info","ts":"2026-11-03T15:34:25.066Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"1f647f19-9640-47d3-8a54-9dd50efdbcc6"}
{"level":"info","ts":"2026-11-03T15:34:25.066Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"1f647f19-9640-47d3-8a54-9dd50efdbcc6","1 head service found":"cluster-status-update-mgqjh-head-svc"}
{"level":"info","ts":"2026-11-03T15:34:25.066Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"1f647f19-9640-47d3-8a54-9dd50efdbcc6"}
{"level":"info","ts":"2026-11-03T15:34:25.114Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"cluster-status-update-mgqjh","reconcileID":"02fbdbf3-7eb3-4bef-a2db-7c0f4712917f","name":"cluster-status-update-mgqjh-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-11-03T15:34:25.115Z","logger":"controllers.NetworkPolicy","msg":"Successfully reconciled NetworkPolicies for RayCluster","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"702036a1-77e4-4ab4-88cf-3b010411228b","cluster":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:34:25.115Z","logger":"controllers.NetworkPolicy","msg":"Reconciling NetworkPolicies for RayCluster","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"50b7f687-22a7-439c-b963-bbc56501c449","cluster":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:34:25.115Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"1f647f19-9640-47d3-8a54-9dd50efdbcc6","error":"certificate ray-head-cert-cluster-status-update-mgqjh not found: certificates.cert-manager.io \"ray-head-cert-cluster-status-update-mgqjh\" not found"}
{"level":"error","ts":"2026-11-03T15:34:25.115Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"1f647f19-9640-47d3-8a54-9dd50efdbcc6","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-cluster-status-update-mgqjh not found: certificates.cert-manager.io \"ray-head-cert-cluster-status-update-mgqjh\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:34:25.115Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"1f647f19-9640-47d3-8a54-9dd50efdbcc6","error":"mTLS secrets not ready: certificate ray-head-cert-cluster-status-update-mgqjh not found: certificates.cert-manager.io \"ray-head-cert-cluster-status-update-mgqjh\" not found"}
{"level":"info","ts":"2026-11-03T15:34:25.115Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"1f647f19-9640-47d3-8a54-9dd50efdbcc6","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"cluster-status-update-mgqjh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:34:25.115Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"1f647f19-9640-47d3-8a54-9dd50efdbcc6","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"cluster-status-update-mgqjh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:34:25.115Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"1f647f19-9640-47d3-8a54-9dd50efdbcc6","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"cluster-status-update-mgqjh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:34:25.115Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"1f647f19-9640-47d3-8a54-9dd50efdbcc6","name":"cluster-status-update-mgqjh","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:34:25.170Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"cluster-status-update-mgqjh","reconcileID":"02fbdbf3-7eb3-4bef-a2db-7c0f4712917f","name":"test-ns-kbx4w-cluster-status-update-mgqjh","operation":"created"}
{"level":"info","ts":"2026-11-03T15:34:25.175Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"0e4b137e-a0ba-49b1-bf96-224cca970a6f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:34:25.175Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"0e4b137e-a0ba-49b1-bf96-224cca970a6f","name":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:34:25.175Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"0e4b137e-a0ba-49b1-bf96-224cca970a6f","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:34:25.175Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"0e4b137e-a0ba-49b1-bf96-224cca970a6f"}
{"level":"info","ts":"2026-11-03T15:34:25.175Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"0e4b137e-a0ba-49b1-bf96-224cca970a6f","RayCluster":"cluster-status-update-mgqjh","State":""}
{"level":"info","ts":"2026-11-03T15:34:25.175Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"0e4b137e-a0ba-49b1-bf96-224cca970a6f","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:34:25.175Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"0e4b137e-a0ba-49b1-bf96-224cca970a6f","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:34:25.176Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"1f647f19-9640-47d3-8a54-9dd50efdbcc6"}
{"level":"error","ts":"2026-11-03T15:34:25.176Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"1f647f19-9640-47d3-8a54-9dd50efdbcc6","error":"mTLS secrets not ready: certificate ray-head-cert-cluster-status-update-mgqjh not found: certificates.cert-manager.io \"ray-head-cert-cluster-status-update-mgqjh\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:34:25.187Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"b92bd4aa-df1b-4cc8-9a56-b6802946d307"}
{"level":"info","ts":"2026-11-03T15:34:25.187Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"b92bd4aa-df1b-4cc8-9a56-b6802946d307"}
{"level":"info","ts":"2026-11-03T15:34:25.187Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"b92bd4aa-df1b-4cc8-9a56-b6802946d307","1 head service found":"cluster-status-update-mgqjh-head-svc"}
{"level":"info","ts":"2026-11-03T15:34:25.187Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"b92bd4aa-df1b-4cc8-9a56-b6802946d307"}
{"level":"info","ts":"2026-11-03T15:34:25.228Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"cluster-status-update-mgqjh","reconcileID":"02fbdbf3-7eb3-4bef-a2db-7c0f4712917f","name":"kube-rbac-proxy-config-cluster-status-update-mgqjh","operation":"created"}
{"level":"info","ts":"2026-11-03T15:34:25.228Z","logger":"controllers.NetworkPolicy","msg":"Successfully reconciled NetworkPolicies for RayCluster","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"50b7f687-22a7-439c-b963-bbc56501c449","cluster":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:34:25.236Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"b92bd4aa-df1b-4cc8-9a56-b6802946d307","error":"certificate ray-head-cert-cluster-status-update-mgqjh not found: certificates.cert-manager.io \"ray-head-cert-cluster-status-update-mgqjh\" not found"}
{"level":"error","ts":"2026-11-03T15:34:25.236Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"b92bd4aa-df1b-4cc8-9a56-b6802946d307","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-cluster-status-update-mgqjh not found: certificates.cert-manager.io \"ray-head-cert-cluster-status-update-mgqjh\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:34:25.236Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"b92bd4aa-df1b-4cc8-9a56-b6802946d307","error":"mTLS secrets not ready: certificate ray-head-cert-cluster-status-update-mgqjh not found: certificates.cert-manager.io \"ray-head-cert-cluster-status-update-mgqjh\" not found"}
{"level":"info","ts":"2026-11-03T15:34:25.236Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"b92bd4aa-df1b-4cc8-9a56-b6802946d307","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"cluster-status-update-mgqjh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:34:25.236Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"b92bd4aa-df1b-4cc8-9a56-b6802946d307","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"cluster-status-update-mgqjh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:34:25.236Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"b92bd4aa-df1b-4cc8-9a56-b6802946d307","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"cluster-status-update-mgqjh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:34:25.236Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"b92bd4aa-df1b-4cc8-9a56-b6802946d307"}
{"level":"error","ts":"2026-11-03T15:34:25.236Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"b92bd4aa-df1b-4cc8-9a56-b6802946d307","error":"mTLS secrets not ready: certificate ray-head-cert-cluster-status-update-mgqjh not found: certificates.cert-manager.io \"ray-head-cert-cluster-status-update-mgqjh\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:34:25.238Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"2428bfe4-4f59-467a-9d76-dc0cd9c8fd30","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:34:25.238Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"2428bfe4-4f59-467a-9d76-dc0cd9c8fd30","name":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:34:25.238Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"2428bfe4-4f59-467a-9d76-dc0cd9c8fd30","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:34:25.238Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"2428bfe4-4f59-467a-9d76-dc0cd9c8fd30"}
{"level":"info","ts":"2026-11-03T15:34:25.238Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"2428bfe4-4f59-467a-9d76-dc0cd9c8fd30","RayCluster":"cluster-status-update-mgqjh","State":""}
{"level":"info","ts":"2026-11-03T15:34:25.238Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"2428bfe4-4f59-467a-9d76-dc0cd9c8fd30","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:34:25.257Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"8fc3598f-b2b0-431e-b475-6ed4249199c3"}
{"level":"info","ts":"2026-11-03T15:34:25.257Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"8fc3598f-b2b0-431e-b475-6ed4249199c3"}
{"level":"info","ts":"2026-11-03T15:34:25.257Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"8fc3598f-b2b0-431e-b475-6ed4249199c3","1 head service found":"cluster-status-update-mgqjh-head-svc"}
{"level":"info","ts":"2026-11-03T15:34:25.257Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"8fc3598f-b2b0-431e-b475-6ed4249199c3"}
{"level":"error","ts":"2026-11-03T15:34:25.282Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"cluster-status-update-mgqjh","reconcileID":"02fbdbf3-7eb3-4bef-a2db-7c0f4712917f","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"cluster-status-update-mgqjh\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:167\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:34:25.282Z","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"cluster-status-update-mgqjh","reconcileID":"02fbdbf3-7eb3-4bef-a2db-7c0f4712917f"}
{"level":"error","ts":"2026-11-03T15:34:25.282Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"cluster-status-update-mgqjh","reconcileID":"02fbdbf3-7eb3-4bef-a2db-7c0f4712917f","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"cluster-status-update-mgqjh\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:34:25.282Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"cluster-status-update-mgqjh","reconcileID":"70ce811f-7f8b-4d64-bade-6090c9ceed11","namespacedName":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:34:25.282Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"cluster-status-update-mgqjh","reconcileID":"70ce811f-7f8b-4d64-bade-6090c9ceed11","mode":"IntegratedOAuth","cluster":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:34:25.282Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"cluster-status-update-mgqjh","reconcileID":"70ce811f-7f8b-4d64-bade-6090c9ceed11","cluster":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:34:25.307Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"8fc3598f-b2b0-431e-b475-6ed4249199c3","error":"certificate ray-head-cert-cluster-status-update-mgqjh not found: certificates.cert-manager.io \"ray-head-cert-cluster-status-update-mgqjh\" not found"}
{"level":"error","ts":"2026-11-03T15:34:25.307Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"8fc3598f-b2b0-431e-b475-6ed4249199c3","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-cluster-status-update-mgqjh not found: certificates.cert-manager.io \"ray-head-cert-cluster-status-update-mgqjh\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:34:25.307Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"8fc3598f-b2b0-431e-b475-6ed4249199c3","error":"mTLS secrets not ready: certificate ray-head-cert-cluster-status-update-mgqjh not found: certificates.cert-manager.io \"ray-head-cert-cluster-status-update-mgqjh\" not found"}
{"level":"info","ts":"2026-11-03T15:34:25.307Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"8fc3598f-b2b0-431e-b475-6ed4249199c3","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"cluster-status-update-mgqjh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:34:25.307Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"8fc3598f-b2b0-431e-b475-6ed4249199c3","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"cluster-status-update-mgqjh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:34:25.307Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"8fc3598f-b2b0-431e-b475-6ed4249199c3","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"cluster-status-update-mgqjh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:34:25.307Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"8fc3598f-b2b0-431e-b475-6ed4249199c3"}
{"level":"error","ts":"2026-11-03T15:34:25.307Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"8fc3598f-b2b0-431e-b475-6ed4249199c3","error":"mTLS secrets not ready: certificate ray-head-cert-cluster-status-update-mgqjh not found: certificates.cert-manager.io \"ray-head-cert-cluster-status-update-mgqjh\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:34:25.343Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"cluster-status-update-mgqjh","reconcileID":"70ce811f-7f8b-4d64-bade-6090c9ceed11","cluster":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:34:25.343Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"3f6bdff4-76eb-47cd-8b21-362b8f06d5c6","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:34:25.343Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"3f6bdff4-76eb-47cd-8b21-362b8f06d5c6","name":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:34:25.343Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"3f6bdff4-76eb-47cd-8b21-362b8f06d5c6","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:34:25.343Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"3f6bdff4-76eb-47cd-8b21-362b8f06d5c6"}
{"level":"info","ts":"2026-11-03T15:34:25.343Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"3f6bdff4-76eb-47cd-8b21-362b8f06d5c6","RayCluster":"cluster-status-update-mgqjh","State":""}
{"level":"info","ts":"2026-11-03T15:34:25.343Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"3f6bdff4-76eb-47cd-8b21-362b8f06d5c6","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:34:25.344Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"3f6bdff4-76eb-47cd-8b21-362b8f06d5c6","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:34:25.347Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"ad50523f-48de-4f5c-9577-969aacadab32"}
{"level":"info","ts":"2026-11-03T15:34:25.347Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"ad50523f-48de-4f5c-9577-969aacadab32"}
{"level":"info","ts":"2026-11-03T15:34:25.347Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"ad50523f-48de-4f5c-9577-969aacadab32","1 head service found":"cluster-status-update-mgqjh-head-svc"}
{"level":"info","ts":"2026-11-03T15:34:25.347Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"ad50523f-48de-4f5c-9577-969aacadab32"}
{"level":"info","ts":"2026-11-03T15:34:25.397Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"ad50523f-48de-4f5c-9577-969aacadab32","error":"certificate ray-head-cert-cluster-status-update-mgqjh not found: certificates.cert-manager.io \"ray-head-cert-cluster-status-update-mgqjh\" not found"}
{"level":"error","ts":"2026-11-03T15:34:25.397Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"ad50523f-48de-4f5c-9577-969aacadab32","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-cluster-status-update-mgqjh not found: certificates.cert-manager.io \"ray-head-cert-cluster-status-update-mgqjh\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:34:25.397Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"ad50523f-48de-4f5c-9577-969aacadab32","error":"mTLS secrets not ready: certificate ray-head-cert-cluster-status-update-mgqjh not found: certificates.cert-manager.io \"ray-head-cert-cluster-status-update-mgqjh\" not found"}
{"level":"info","ts":"2026-11-03T15:34:25.397Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"ad50523f-48de-4f5c-9577-969aacadab32","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"cluster-status-update-mgqjh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:34:25.397Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"ad50523f-48de-4f5c-9577-969aacadab32","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"cluster-status-update-mgqjh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:34:25.397Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"ad50523f-48de-4f5c-9577-969aacadab32","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"cluster-status-update-mgqjh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:34:25.397Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"ad50523f-48de-4f5c-9577-969aacadab32"}
{"level":"error","ts":"2026-11-03T15:34:25.397Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"ad50523f-48de-4f5c-9577-969aacadab32","error":"mTLS secrets not ready: certificate ray-head-cert-cluster-status-update-mgqjh not found: certificates.cert-manager.io \"ray-head-cert-cluster-status-update-mgqjh\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:34:25.404Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"1628ead8-c47d-466e-8c3a-7c976a71caf2","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:34:25.404Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"1628ead8-c47d-466e-8c3a-7c976a71caf2","name":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:34:25.404Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"1628ead8-c47d-466e-8c3a-7c976a71caf2","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:34:25.404Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"1628ead8-c47d-466e-8c3a-7c976a71caf2"}
{"level":"info","ts":"2026-11-03T15:34:25.404Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"1628ead8-c47d-466e-8c3a-7c976a71caf2","RayCluster":"cluster-status-update-mgqjh","State":""}
{"level":"info","ts":"2026-11-03T15:34:25.404Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"1628ead8-c47d-466e-8c3a-7c976a71caf2","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:34:25.478Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"93b39ad4-48e7-441a-b5e9-d8cdf7f037e0"}
{"level":"info","ts":"2026-11-03T15:34:25.478Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"93b39ad4-48e7-441a-b5e9-d8cdf7f037e0"}
{"level":"info","ts":"2026-11-03T15:34:25.478Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"93b39ad4-48e7-441a-b5e9-d8cdf7f037e0","1 head service found":"cluster-status-update-mgqjh-head-svc"}
{"level":"info","ts":"2026-11-03T15:34:25.478Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"93b39ad4-48e7-441a-b5e9-d8cdf7f037e0"}
{"level":"info","ts":"2026-11-03T15:34:25.527Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"93b39ad4-48e7-441a-b5e9-d8cdf7f037e0","error":"certificate ray-head-cert-cluster-status-update-mgqjh not found: certificates.cert-manager.io \"ray-head-cert-cluster-status-update-mgqjh\" not found"}
{"level":"error","ts":"2026-11-03T15:34:25.527Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"93b39ad4-48e7-441a-b5e9-d8cdf7f037e0","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-cluster-status-update-mgqjh not found: certificates.cert-manager.io \"ray-head-cert-cluster-status-update-mgqjh\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:34:25.527Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"93b39ad4-48e7-441a-b5e9-d8cdf7f037e0","error":"mTLS secrets not ready: certificate ray-head-cert-cluster-status-update-mgqjh not found: certificates.cert-manager.io \"ray-head-cert-cluster-status-update-mgqjh\" not found"}
{"level":"info","ts":"2026-11-03T15:34:25.527Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"93b39ad4-48e7-441a-b5e9-d8cdf7f037e0","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"cluster-status-update-mgqjh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:34:25.527Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"93b39ad4-48e7-441a-b5e9-d8cdf7f037e0","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"cluster-status-update-mgqjh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:34:25.528Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"93b39ad4-48e7-441a-b5e9-d8cdf7f037e0","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"cluster-status-update-mgqjh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:34:25.528Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"93b39ad4-48e7-441a-b5e9-d8cdf7f037e0"}
{"level":"error","ts":"2026-11-03T15:34:25.528Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"93b39ad4-48e7-441a-b5e9-d8cdf7f037e0","error":"mTLS secrets not ready: certificate ray-head-cert-cluster-status-update-mgqjh not found: certificates.cert-manager.io \"ray-head-cert-cluster-status-update-mgqjh\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:34:25.689Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"07fe6443-27ce-4f61-a8fe-6b1050ec3dce"}
{"level":"info","ts":"2026-11-03T15:34:25.689Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"07fe6443-27ce-4f61-a8fe-6b1050ec3dce"}
{"level":"info","ts":"2026-11-03T15:34:25.689Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"07fe6443-27ce-4f61-a8fe-6b1050ec3dce","1 head service found":"cluster-status-update-mgqjh-head-svc"}
{"level":"info","ts":"2026-11-03T15:34:25.689Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"07fe6443-27ce-4f61-a8fe-6b1050ec3dce"}
{"level":"info","ts":"2026-11-03T15:34:25.739Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"07fe6443-27ce-4f61-a8fe-6b1050ec3dce","error":"certificate ray-head-cert-cluster-status-update-mgqjh is not ready at current generation 1"}
{"level":"error","ts":"2026-11-03T15:34:25.739Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"07fe6443-27ce-4f61-a8fe-6b1050ec3dce","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-cluster-status-update-mgqjh is not ready at current generation 1","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:34:25.739Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"07fe6443-27ce-4f61-a8fe-6b1050ec3dce","error":"mTLS secrets not ready: certificate ray-head-cert-cluster-status-update-mgqjh is not ready at current generation 1"}
{"level":"info","ts":"2026-11-03T15:34:25.739Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"07fe6443-27ce-4f61-a8fe-6b1050ec3dce","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"cluster-status-update-mgqjh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:34:25.739Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"07fe6443-27ce-4f61-a8fe-6b1050ec3dce","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"cluster-status-update-mgqjh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:34:25.739Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"07fe6443-27ce-4f61-a8fe-6b1050ec3dce","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"cluster-status-update-mgqjh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:34:25.739Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"07fe6443-27ce-4f61-a8fe-6b1050ec3dce"}
{"level":"error","ts":"2026-11-03T15:34:25.739Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"07fe6443-27ce-4f61-a8fe-6b1050ec3dce","error":"mTLS secrets not ready: certificate ray-head-cert-cluster-status-update-mgqjh is not ready at current generation 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:34:25.883Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"388dff49-1681-4a8b-ab5b-4b75a6264be5","certificate":"ray-head-cert-cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:34:25.883Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"388dff49-1681-4a8b-ab5b-4b75a6264be5"}
{"level":"info","ts":"2026-11-03T15:34:25.883Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"95000604-61e3-43fd-8d93-c1b19e66130d","cluster":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:34:26.061Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"65bb6c50-100f-457e-9597-60c3ad791fe6"}
{"level":"info","ts":"2026-11-03T15:34:26.061Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"65bb6c50-100f-457e-9597-60c3ad791fe6"}
{"level":"info","ts":"2026-11-03T15:34:26.061Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"65bb6c50-100f-457e-9597-60c3ad791fe6","1 head service found":"cluster-status-update-mgqjh-head-svc"}
{"level":"info","ts":"2026-11-03T15:34:26.061Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"65bb6c50-100f-457e-9597-60c3ad791fe6"}
{"level":"info","ts":"2026-11-03T15:34:26.113Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"65bb6c50-100f-457e-9597-60c3ad791fe6","error":"certificate ray-head-cert-cluster-status-update-mgqjh is not ready at current generation 1"}
{"level":"error","ts":"2026-11-03T15:34:26.113Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"65bb6c50-100f-457e-9597-60c3ad791fe6","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-cluster-status-update-mgqjh is not ready at current generation 1","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:34:26.113Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"65bb6c50-100f-457e-9597-60c3ad791fe6","error":"mTLS secrets not ready: certificate ray-head-cert-cluster-status-update-mgqjh is not ready at current generation 1"}
{"level":"info","ts":"2026-11-03T15:34:26.113Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"65bb6c50-100f-457e-9597-60c3ad791fe6","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"cluster-status-update-mgqjh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:34:26.113Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"65bb6c50-100f-457e-9597-60c3ad791fe6","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"cluster-status-update-mgqjh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:34:26.113Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"65bb6c50-100f-457e-9597-60c3ad791fe6","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"cluster-status-update-mgqjh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:34:26.113Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"65bb6c50-100f-457e-9597-60c3ad791fe6"}
{"level":"error","ts":"2026-11-03T15:34:26.113Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"65bb6c50-100f-457e-9597-60c3ad791fe6","error":"mTLS secrets not ready: certificate ray-head-cert-cluster-status-update-mgqjh is not ready at current generation 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:34:26.337Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"95000604-61e3-43fd-8d93-c1b19e66130d","certificate":"ray-head-cert-cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:34:26.337Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"95000604-61e3-43fd-8d93-c1b19e66130d"}
{"level":"info","ts":"2026-11-03T15:34:26.754Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"cb3abbe8-2bf0-423f-af7a-1abf0ec63abf"}
{"level":"info","ts":"2026-11-03T15:34:26.754Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"cb3abbe8-2bf0-423f-af7a-1abf0ec63abf"}
{"level":"info","ts":"2026-11-03T15:34:26.754Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"cb3abbe8-2bf0-423f-af7a-1abf0ec63abf","1 head service found":"cluster-status-update-mgqjh-head-svc"}
{"level":"info","ts":"2026-11-03T15:34:26.754Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"cb3abbe8-2bf0-423f-af7a-1abf0ec63abf"}
{"level":"info","ts":"2026-11-03T15:34:26.806Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"cb3abbe8-2bf0-423f-af7a-1abf0ec63abf","error":"certificate ray-head-cert-cluster-status-update-mgqjh is not ready at current generation 1"}
{"level":"error","ts":"2026-11-03T15:34:26.806Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"cb3abbe8-2bf0-423f-af7a-1abf0ec63abf","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-cluster-status-update-mgqjh is not ready at current generation 1","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:34:26.806Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"cb3abbe8-2bf0-423f-af7a-1abf0ec63abf","error":"mTLS secrets not ready: certificate ray-head-cert-cluster-status-update-mgqjh is not ready at current generation 1"}
{"level":"info","ts":"2026-11-03T15:34:26.807Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"cb3abbe8-2bf0-423f-af7a-1abf0ec63abf","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"cluster-status-update-mgqjh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:34:26.807Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"cb3abbe8-2bf0-423f-af7a-1abf0ec63abf","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"cluster-status-update-mgqjh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:34:26.807Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"cb3abbe8-2bf0-423f-af7a-1abf0ec63abf","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"cluster-status-update-mgqjh","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:34:26.807Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"cb3abbe8-2bf0-423f-af7a-1abf0ec63abf"}
{"level":"error","ts":"2026-11-03T15:34:26.807Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"cb3abbe8-2bf0-423f-af7a-1abf0ec63abf","error":"mTLS secrets not ready: certificate ray-head-cert-cluster-status-update-mgqjh is not ready at current generation 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:34:28.087Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"6d0240d9-96f5-44fe-8916-1d76abc0816a"}
{"level":"info","ts":"2026-11-03T15:34:28.087Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"6d0240d9-96f5-44fe-8916-1d76abc0816a"}
{"level":"info","ts":"2026-11-03T15:34:28.087Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"6d0240d9-96f5-44fe-8916-1d76abc0816a","1 head service found":"cluster-status-update-mgqjh-head-svc"}
{"level":"info","ts":"2026-11-03T15:34:28.088Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"6d0240d9-96f5-44fe-8916-1d76abc0816a"}
{"level":"info","ts":"2026-11-03T15:34:28.189Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"6d0240d9-96f5-44fe-8916-1d76abc0816a","mode":"IntegratedOAuth","cluster":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:34:28.189Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"6d0240d9-96f5-44fe-8916-1d76abc0816a","cluster":"cluster-status-update-mgqjh","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-11-03T15:34:28.189Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"6d0240d9-96f5-44fe-8916-1d76abc0816a","cluster":"cluster-status-update-mgqjh","authType":"IntegratedOAuth","serviceAccount":"cluster-status-update-mgqjh-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-11-03T15:34:28.189Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"6d0240d9-96f5-44fe-8916-1d76abc0816a","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"cluster-status-update-mgqjh","ray.io/group":"headgroup","ray.io/identifier":"cluster-status-update-mgqjh-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-11-03T15:34:28.189Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"6d0240d9-96f5-44fe-8916-1d76abc0816a","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-11-03T15:34:28.189Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"6d0240d9-96f5-44fe-8916-1d76abc0816a","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-11-03T15:34:28.189Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"6d0240d9-96f5-44fe-8916-1d76abc0816a","rayNodeType":"head","generatedCmd":"ulimit -n ${RAY_START_ULIMIT_OPEN_FILES:-65536}; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-11-03T15:34:28.189Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"6d0240d9-96f5-44fe-8916-1d76abc0816a","enabled":true}
{"level":"info","ts":"2026-11-03T15:34:28.262Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"6d0240d9-96f5-44fe-8916-1d76abc0816a","name":"cluster-status-update-mgqjh-head-5wcnb"}
{"level":"info","ts":"2026-11-03T15:34:28.262Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"6d0240d9-96f5-44fe-8916-1d76abc0816a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:34:28.262Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"6d0240d9-96f5-44fe-8916-1d76abc0816a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:34:28.262Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"6d0240d9-96f5-44fe-8916-1d76abc0816a","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-11-03T15:34:28.262Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"6d0240d9-96f5-44fe-8916-1d76abc0816a","group":"small-group","indices":{}}
{"level":"info","ts":"2026-11-03T15:34:28.263Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"6d0240d9-96f5-44fe-8916-1d76abc0816a","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-11-03T15:34:28.405Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"90c5ba31-8054-49b3-8e3a-5da066029587","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:34:28.405Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"90c5ba31-8054-49b3-8e3a-5da066029587","name":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:34:28.405Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"90c5ba31-8054-49b3-8e3a-5da066029587","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:34:28.405Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"90c5ba31-8054-49b3-8e3a-5da066029587"}
{"level":"info","ts":"2026-11-03T15:34:28.405Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"90c5ba31-8054-49b3-8e3a-5da066029587","RayCluster":"cluster-status-update-mgqjh","State":""}
{"level":"info","ts":"2026-11-03T15:34:28.405Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"90c5ba31-8054-49b3-8e3a-5da066029587","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:34:28.409Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"6d0240d9-96f5-44fe-8916-1d76abc0816a","creating worker for group":"small-group","index":0,"total":1,"replicaIndex":0}
{"level":"info","ts":"2026-11-03T15:34:28.409Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"6d0240d9-96f5-44fe-8916-1d76abc0816a","nodeType":"worker","rayStartParams":{"address":"cluster-status-update-mgqjh-head-svc.test-ns-kbx4w.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-11-03T15:34:28.409Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"6d0240d9-96f5-44fe-8916-1d76abc0816a","rayStartCmd":"ray start  --address=cluster-status-update-mgqjh-head-svc.test-ns-kbx4w.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-11-03T15:34:28.409Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"6d0240d9-96f5-44fe-8916-1d76abc0816a","rayNodeType":"worker","generatedCmd":"ulimit -n ${RAY_START_ULIMIT_OPEN_FILES:-65536}; ray start  --address=cluster-status-update-mgqjh-head-svc.test-ns-kbx4w.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-11-03T15:34:28.409Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"6d0240d9-96f5-44fe-8916-1d76abc0816a","enabled":true}
{"level":"info","ts":"2026-11-03T15:34:28.471Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"6d0240d9-96f5-44fe-8916-1d76abc0816a","name":"cluster-status-update-mgqjh-small-group-worker-d5k5g"}
{"level":"info","ts":"2026-11-03T15:34:28.471Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"6d0240d9-96f5-44fe-8916-1d76abc0816a","podName":"cluster-status-update-mgqjh-head-5wcnb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:34:28.471Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"6d0240d9-96f5-44fe-8916-1d76abc0816a","podName":"cluster-status-update-mgqjh-head-5wcnb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:34:28.471Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"6d0240d9-96f5-44fe-8916-1d76abc0816a","name":"cluster-status-update-mgqjh","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:28Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:34:28.532Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"8111d80e-e9f8-4b44-b395-c2958840d46b","cluster":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:34:28.532Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"a5957d1d-1f97-4b5d-b0af-4af69adfca0c"}
{"level":"info","ts":"2026-11-03T15:34:28.533Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"a5957d1d-1f97-4b5d-b0af-4af69adfca0c"}
{"level":"info","ts":"2026-11-03T15:34:28.533Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"a5957d1d-1f97-4b5d-b0af-4af69adfca0c","1 head service found":"cluster-status-update-mgqjh-head-svc"}
{"level":"info","ts":"2026-11-03T15:34:28.533Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"50a652d8-ec02-475f-afe0-8de284aa806c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:34:28.533Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"50a652d8-ec02-475f-afe0-8de284aa806c","name":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:34:28.533Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"a5957d1d-1f97-4b5d-b0af-4af69adfca0c","Found 1 head Pod":"cluster-status-update-mgqjh-head-5wcnb","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:34:28.533Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"a5957d1d-1f97-4b5d-b0af-4af69adfca0c","head Pod":"cluster-status-update-mgqjh-head-5wcnb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod cluster-status-update-mgqjh-head-5wcnb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:34:28.533Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"50a652d8-ec02-475f-afe0-8de284aa806c","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:34:28.533Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"a5957d1d-1f97-4b5d-b0af-4af69adfca0c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:34:28.533Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"a5957d1d-1f97-4b5d-b0af-4af69adfca0c","worker Pod":"cluster-status-update-mgqjh-small-group-worker-d5k5g","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod cluster-status-update-mgqjh-small-group-worker-d5k5g. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:34:28.533Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"a5957d1d-1f97-4b5d-b0af-4af69adfca0c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:34:28.533Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"a5957d1d-1f97-4b5d-b0af-4af69adfca0c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:34:28.533Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"a5957d1d-1f97-4b5d-b0af-4af69adfca0c","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:34:28.533Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"a5957d1d-1f97-4b5d-b0af-4af69adfca0c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:34:28.533Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"a5957d1d-1f97-4b5d-b0af-4af69adfca0c","podName":"cluster-status-update-mgqjh-head-5wcnb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:34:28.533Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"a5957d1d-1f97-4b5d-b0af-4af69adfca0c","podName":"cluster-status-update-mgqjh-head-5wcnb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:34:28.533Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"50a652d8-ec02-475f-afe0-8de284aa806c"}
{"level":"info","ts":"2026-11-03T15:34:28.533Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"50a652d8-ec02-475f-afe0-8de284aa806c","RayCluster":"cluster-status-update-mgqjh","State":""}
{"level":"info","ts":"2026-11-03T15:34:28.533Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"50a652d8-ec02-475f-afe0-8de284aa806c","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:28Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:34:28.533Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"50a652d8-ec02-475f-afe0-8de284aa806c","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:34:28.533Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"a5957d1d-1f97-4b5d-b0af-4af69adfca0c","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:34:28.533Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"a5957d1d-1f97-4b5d-b0af-4af69adfca0c","seconds":300}
{"level":"info","ts":"2026-11-03T15:34:28.549Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"8fc37a41-4824-4b1a-834b-039b9575d446"}
{"level":"info","ts":"2026-11-03T15:34:28.549Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"8fc37a41-4824-4b1a-834b-039b9575d446"}
{"level":"info","ts":"2026-11-03T15:34:28.549Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"8fc37a41-4824-4b1a-834b-039b9575d446","1 head service found":"cluster-status-update-mgqjh-head-svc"}
{"level":"info","ts":"2026-11-03T15:34:28.549Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"8fc37a41-4824-4b1a-834b-039b9575d446","Found 1 head Pod":"cluster-status-update-mgqjh-head-5wcnb","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:34:28.549Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"8fc37a41-4824-4b1a-834b-039b9575d446","head Pod":"cluster-status-update-mgqjh-head-5wcnb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod cluster-status-update-mgqjh-head-5wcnb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:34:28.549Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"8fc37a41-4824-4b1a-834b-039b9575d446","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:34:28.549Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"8fc37a41-4824-4b1a-834b-039b9575d446","worker Pod":"cluster-status-update-mgqjh-small-group-worker-d5k5g","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod cluster-status-update-mgqjh-small-group-worker-d5k5g. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:34:28.549Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"8fc37a41-4824-4b1a-834b-039b9575d446","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:34:28.549Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"8fc37a41-4824-4b1a-834b-039b9575d446","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:34:28.549Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"8fc37a41-4824-4b1a-834b-039b9575d446","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:34:28.549Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"8fc37a41-4824-4b1a-834b-039b9575d446","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:34:28.549Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"8fc37a41-4824-4b1a-834b-039b9575d446","podName":"cluster-status-update-mgqjh-head-5wcnb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:34:28.549Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"8fc37a41-4824-4b1a-834b-039b9575d446","podName":"cluster-status-update-mgqjh-head-5wcnb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:34:28.549Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"8fc37a41-4824-4b1a-834b-039b9575d446","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:34:28.549Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"8fc37a41-4824-4b1a-834b-039b9575d446","seconds":300}
{"level":"info","ts":"2026-11-03T15:34:28.598Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"7448f343-32d2-4395-995c-023ad6a77864","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:34:28.598Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"7448f343-32d2-4395-995c-023ad6a77864","name":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:34:28.598Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"7448f343-32d2-4395-995c-023ad6a77864","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:34:28.598Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"7448f343-32d2-4395-995c-023ad6a77864"}
{"level":"info","ts":"2026-11-03T15:34:28.598Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"7448f343-32d2-4395-995c-023ad6a77864","RayCluster":"cluster-status-update-mgqjh","State":""}
{"level":"info","ts":"2026-11-03T15:34:28.598Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"7448f343-32d2-4395-995c-023ad6a77864","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:28Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:28Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:34:28.605Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"f933eebf-0a3b-4573-95f0-4a385e756cf3"}
{"level":"info","ts":"2026-11-03T15:34:28.605Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"f933eebf-0a3b-4573-95f0-4a385e756cf3"}
{"level":"info","ts":"2026-11-03T15:34:28.605Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"f933eebf-0a3b-4573-95f0-4a385e756cf3","1 head service found":"cluster-status-update-mgqjh-head-svc"}
{"level":"info","ts":"2026-11-03T15:34:28.605Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"f933eebf-0a3b-4573-95f0-4a385e756cf3","Found 1 head Pod":"cluster-status-update-mgqjh-head-5wcnb","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:34:28.605Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"f933eebf-0a3b-4573-95f0-4a385e756cf3","head Pod":"cluster-status-update-mgqjh-head-5wcnb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod cluster-status-update-mgqjh-head-5wcnb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:34:28.605Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"f933eebf-0a3b-4573-95f0-4a385e756cf3","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:34:28.605Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"f933eebf-0a3b-4573-95f0-4a385e756cf3","worker Pod":"cluster-status-update-mgqjh-small-group-worker-d5k5g","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod cluster-status-update-mgqjh-small-group-worker-d5k5g. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:34:28.605Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"f933eebf-0a3b-4573-95f0-4a385e756cf3","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:34:28.605Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"f933eebf-0a3b-4573-95f0-4a385e756cf3","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:34:28.605Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"f933eebf-0a3b-4573-95f0-4a385e756cf3","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:34:28.605Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"f933eebf-0a3b-4573-95f0-4a385e756cf3","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:34:28.605Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"f933eebf-0a3b-4573-95f0-4a385e756cf3","podName":"cluster-status-update-mgqjh-head-5wcnb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:34:28.605Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"f933eebf-0a3b-4573-95f0-4a385e756cf3","podName":"cluster-status-update-mgqjh-head-5wcnb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:34:28.605Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"f933eebf-0a3b-4573-95f0-4a385e756cf3","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:34:28.605Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"f933eebf-0a3b-4573-95f0-4a385e756cf3","seconds":300}
{"level":"info","ts":"2026-11-03T15:34:29.017Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"87011dc6-605e-4b83-ac14-9a8d248433eb"}
{"level":"info","ts":"2026-11-03T15:34:29.017Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"87011dc6-605e-4b83-ac14-9a8d248433eb"}
{"level":"info","ts":"2026-11-03T15:34:29.017Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"87011dc6-605e-4b83-ac14-9a8d248433eb","1 head service found":"cluster-status-update-mgqjh-head-svc"}
{"level":"info","ts":"2026-11-03T15:34:29.017Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"87011dc6-605e-4b83-ac14-9a8d248433eb","Found 1 head Pod":"cluster-status-update-mgqjh-head-5wcnb","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:34:29.017Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"87011dc6-605e-4b83-ac14-9a8d248433eb","head Pod":"cluster-status-update-mgqjh-head-5wcnb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod cluster-status-update-mgqjh-head-5wcnb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:34:29.017Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"87011dc6-605e-4b83-ac14-9a8d248433eb","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:34:29.017Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"87011dc6-605e-4b83-ac14-9a8d248433eb","worker Pod":"cluster-status-update-mgqjh-small-group-worker-d5k5g","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod cluster-status-update-mgqjh-small-group-worker-d5k5g. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:34:29.017Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"87011dc6-605e-4b83-ac14-9a8d248433eb","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:34:29.017Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"87011dc6-605e-4b83-ac14-9a8d248433eb","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:34:29.017Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"87011dc6-605e-4b83-ac14-9a8d248433eb","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:34:29.017Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"87011dc6-605e-4b83-ac14-9a8d248433eb","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:34:29.017Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"87011dc6-605e-4b83-ac14-9a8d248433eb","podName":"cluster-status-update-mgqjh-head-5wcnb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:34:29.017Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"87011dc6-605e-4b83-ac14-9a8d248433eb","podName":"cluster-status-update-mgqjh-head-5wcnb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:34:29.017Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"87011dc6-605e-4b83-ac14-9a8d248433eb","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:34:29.017Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"87011dc6-605e-4b83-ac14-9a8d248433eb","seconds":300}
{"level":"info","ts":"2026-11-03T15:34:29.230Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"f1584466-2ae7-4d53-91fa-dbb1ac5589a6"}
{"level":"info","ts":"2026-11-03T15:34:29.231Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"f1584466-2ae7-4d53-91fa-dbb1ac5589a6"}
{"level":"info","ts":"2026-11-03T15:34:29.231Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"f1584466-2ae7-4d53-91fa-dbb1ac5589a6","1 head service found":"cluster-status-update-mgqjh-head-svc"}
{"level":"info","ts":"2026-11-03T15:34:29.231Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"f1584466-2ae7-4d53-91fa-dbb1ac5589a6","Found 1 head Pod":"cluster-status-update-mgqjh-head-5wcnb","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:34:29.231Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"f1584466-2ae7-4d53-91fa-dbb1ac5589a6","head Pod":"cluster-status-update-mgqjh-head-5wcnb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod cluster-status-update-mgqjh-head-5wcnb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:34:29.231Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"f1584466-2ae7-4d53-91fa-dbb1ac5589a6","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:34:29.231Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"f1584466-2ae7-4d53-91fa-dbb1ac5589a6","worker Pod":"cluster-status-update-mgqjh-small-group-worker-d5k5g","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod cluster-status-update-mgqjh-small-group-worker-d5k5g. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:34:29.231Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"f1584466-2ae7-4d53-91fa-dbb1ac5589a6","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:34:29.231Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"f1584466-2ae7-4d53-91fa-dbb1ac5589a6","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:34:29.231Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"f1584466-2ae7-4d53-91fa-dbb1ac5589a6","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:34:29.231Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"f1584466-2ae7-4d53-91fa-dbb1ac5589a6","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:34:29.231Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"f1584466-2ae7-4d53-91fa-dbb1ac5589a6","podName":"cluster-status-update-mgqjh-head-5wcnb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:34:29.231Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"f1584466-2ae7-4d53-91fa-dbb1ac5589a6","podName":"cluster-status-update-mgqjh-head-5wcnb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:34:29.231Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"f1584466-2ae7-4d53-91fa-dbb1ac5589a6","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:34:29.231Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"f1584466-2ae7-4d53-91fa-dbb1ac5589a6","seconds":300}
{"level":"info","ts":"2026-11-03T15:34:29.675Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"13d4507a-0097-496a-8199-fc89a911b3ad"}
{"level":"info","ts":"2026-11-03T15:34:29.675Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"13d4507a-0097-496a-8199-fc89a911b3ad"}
{"level":"info","ts":"2026-11-03T15:34:29.675Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"13d4507a-0097-496a-8199-fc89a911b3ad","1 head service found":"cluster-status-update-mgqjh-head-svc"}
{"level":"info","ts":"2026-11-03T15:34:29.675Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"13d4507a-0097-496a-8199-fc89a911b3ad","Found 1 head Pod":"cluster-status-update-mgqjh-head-5wcnb","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:34:29.675Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"13d4507a-0097-496a-8199-fc89a911b3ad","head Pod":"cluster-status-update-mgqjh-head-5wcnb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod cluster-status-update-mgqjh-head-5wcnb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:34:29.675Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"13d4507a-0097-496a-8199-fc89a911b3ad","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:34:29.675Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"13d4507a-0097-496a-8199-fc89a911b3ad","worker Pod":"cluster-status-update-mgqjh-small-group-worker-d5k5g","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod cluster-status-update-mgqjh-small-group-worker-d5k5g. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:34:29.675Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"13d4507a-0097-496a-8199-fc89a911b3ad","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:34:29.675Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"13d4507a-0097-496a-8199-fc89a911b3ad","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:34:29.675Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"13d4507a-0097-496a-8199-fc89a911b3ad","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:34:29.675Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"13d4507a-0097-496a-8199-fc89a911b3ad","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:34:29.675Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"13d4507a-0097-496a-8199-fc89a911b3ad","podName":"cluster-status-update-mgqjh-head-5wcnb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:34:29.675Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"13d4507a-0097-496a-8199-fc89a911b3ad","podName":"cluster-status-update-mgqjh-head-5wcnb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:34:29.675Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"13d4507a-0097-496a-8199-fc89a911b3ad","name":"cluster-status-update-mgqjh","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:28Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:34:29.738Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"38e7376e-41ad-4b9c-979b-2d627fa4a3e2","cluster":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:34:29.738Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"5c1e0479-7afe-46fa-837b-2da4782fdb71","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:34:29.738Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"5c1e0479-7afe-46fa-837b-2da4782fdb71","name":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:34:29.738Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"5c1e0479-7afe-46fa-837b-2da4782fdb71","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:34:29.738Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"5c1e0479-7afe-46fa-837b-2da4782fdb71"}
{"level":"info","ts":"2026-11-03T15:34:29.738Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"5c1e0479-7afe-46fa-837b-2da4782fdb71","RayCluster":"cluster-status-update-mgqjh","State":""}
{"level":"info","ts":"2026-11-03T15:34:29.738Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"5c1e0479-7afe-46fa-837b-2da4782fdb71","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:28Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:34:29.738Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"5c1e0479-7afe-46fa-837b-2da4782fdb71","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:34:29.799Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"50a870a9-9c07-46a3-ac0f-499f86dfb53d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:34:29.799Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"50a870a9-9c07-46a3-ac0f-499f86dfb53d","name":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:34:29.799Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"50a870a9-9c07-46a3-ac0f-499f86dfb53d","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:34:29.800Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"50a870a9-9c07-46a3-ac0f-499f86dfb53d"}
{"level":"info","ts":"2026-11-03T15:34:29.800Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"50a870a9-9c07-46a3-ac0f-499f86dfb53d","RayCluster":"cluster-status-update-mgqjh","State":""}
{"level":"info","ts":"2026-11-03T15:34:29.800Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"50a870a9-9c07-46a3-ac0f-499f86dfb53d","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:34:29.894Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"707f4ecb-c2ce-41c4-922a-947130d8f441"}
{"level":"info","ts":"2026-11-03T15:34:29.894Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"707f4ecb-c2ce-41c4-922a-947130d8f441"}
{"level":"info","ts":"2026-11-03T15:34:29.894Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"707f4ecb-c2ce-41c4-922a-947130d8f441","1 head service found":"cluster-status-update-mgqjh-head-svc"}
{"level":"info","ts":"2026-11-03T15:34:29.894Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"707f4ecb-c2ce-41c4-922a-947130d8f441","Found 1 head Pod":"cluster-status-update-mgqjh-head-5wcnb","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:34:29.894Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"707f4ecb-c2ce-41c4-922a-947130d8f441","head Pod":"cluster-status-update-mgqjh-head-5wcnb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod cluster-status-update-mgqjh-head-5wcnb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:34:29.894Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"707f4ecb-c2ce-41c4-922a-947130d8f441","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:34:29.894Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"707f4ecb-c2ce-41c4-922a-947130d8f441","worker Pod":"cluster-status-update-mgqjh-small-group-worker-d5k5g","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod cluster-status-update-mgqjh-small-group-worker-d5k5g. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:34:29.894Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"707f4ecb-c2ce-41c4-922a-947130d8f441","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:34:29.894Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"707f4ecb-c2ce-41c4-922a-947130d8f441","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:34:29.894Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"707f4ecb-c2ce-41c4-922a-947130d8f441","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:34:29.894Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"707f4ecb-c2ce-41c4-922a-947130d8f441","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:34:29.894Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"707f4ecb-c2ce-41c4-922a-947130d8f441","podName":"cluster-status-update-mgqjh-head-5wcnb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:34:29.894Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"707f4ecb-c2ce-41c4-922a-947130d8f441","podName":"cluster-status-update-mgqjh-head-5wcnb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:34:29.894Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"707f4ecb-c2ce-41c4-922a-947130d8f441","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:34:29.894Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"707f4ecb-c2ce-41c4-922a-947130d8f441","seconds":300}
{"level":"info","ts":"2026-11-03T15:34:30.041Z","logger":"controllers.RayClusterMTLS","msg":"Updating head certificate","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"38e7376e-41ad-4b9c-979b-2d627fa4a3e2","certificate":"ray-head-cert-cluster-status-update-mgqjh","ipAddresses":["10.132.0.47","127.0.0.1"]}
{"level":"info","ts":"2026-11-03T15:34:30.303Z","logger":"controllers.RayClusterMTLS","msg":"Updating worker certificate","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"38e7376e-41ad-4b9c-979b-2d627fa4a3e2","certificate":"ray-worker-cert-cluster-status-update-mgqjh","ipAddresses":["10.134.0.51","127.0.0.1"]}
{"level":"info","ts":"2026-11-03T15:34:30.564Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"38e7376e-41ad-4b9c-979b-2d627fa4a3e2","certificate":"ray-head-cert-cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:34:30.564Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"38e7376e-41ad-4b9c-979b-2d627fa4a3e2"}
{"level":"info","ts":"2026-11-03T15:34:32.800Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"98864be1-f4f3-47e3-a8ec-c0cda3e79258","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:34:32.800Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"98864be1-f4f3-47e3-a8ec-c0cda3e79258","name":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:34:32.800Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"98864be1-f4f3-47e3-a8ec-c0cda3e79258","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:34:32.801Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"98864be1-f4f3-47e3-a8ec-c0cda3e79258"}
{"level":"info","ts":"2026-11-03T15:34:32.801Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"98864be1-f4f3-47e3-a8ec-c0cda3e79258","RayCluster":"cluster-status-update-mgqjh","State":""}
{"level":"info","ts":"2026-11-03T15:34:32.801Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"98864be1-f4f3-47e3-a8ec-c0cda3e79258","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:34:35.801Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"11184064-f825-4e57-bcc5-6b446587e03e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:34:35.801Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"11184064-f825-4e57-bcc5-6b446587e03e","name":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:34:35.801Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"11184064-f825-4e57-bcc5-6b446587e03e","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:34:35.802Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"11184064-f825-4e57-bcc5-6b446587e03e"}
{"level":"info","ts":"2026-11-03T15:34:35.802Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"11184064-f825-4e57-bcc5-6b446587e03e","RayCluster":"cluster-status-update-mgqjh","State":""}
{"level":"info","ts":"2026-11-03T15:34:35.802Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"11184064-f825-4e57-bcc5-6b446587e03e","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:34:38.803Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"99dde446-566f-4a07-aec7-171da058fdca","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:34:38.803Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"99dde446-566f-4a07-aec7-171da058fdca","name":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:34:38.803Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"99dde446-566f-4a07-aec7-171da058fdca","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:34:38.803Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"99dde446-566f-4a07-aec7-171da058fdca"}
{"level":"info","ts":"2026-11-03T15:34:38.803Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"99dde446-566f-4a07-aec7-171da058fdca","RayCluster":"cluster-status-update-mgqjh","State":""}
{"level":"info","ts":"2026-11-03T15:34:38.803Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"99dde446-566f-4a07-aec7-171da058fdca","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:34:39.637Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"f7b576de-6953-457a-a75b-29e633225f1d"}
{"level":"info","ts":"2026-11-03T15:34:39.637Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"f7b576de-6953-457a-a75b-29e633225f1d"}
{"level":"info","ts":"2026-11-03T15:34:39.637Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"f7b576de-6953-457a-a75b-29e633225f1d","1 head service found":"long-running-hvml9-head-svc"}
{"level":"info","ts":"2026-11-03T15:34:39.637Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"f7b576de-6953-457a-a75b-29e633225f1d","Found 1 head Pod":"long-running-hvml9-head-s8j5v","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:34:39.637Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"f7b576de-6953-457a-a75b-29e633225f1d","head Pod":"long-running-hvml9-head-s8j5v","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod long-running-hvml9-head-s8j5v. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:34:39.637Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"f7b576de-6953-457a-a75b-29e633225f1d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:34:39.638Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"f7b576de-6953-457a-a75b-29e633225f1d","worker Pod":"long-running-hvml9-small-group-worker-mjl4n","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod long-running-hvml9-small-group-worker-mjl4n. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:34:39.638Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"f7b576de-6953-457a-a75b-29e633225f1d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:34:39.638Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"f7b576de-6953-457a-a75b-29e633225f1d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:34:39.638Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"f7b576de-6953-457a-a75b-29e633225f1d","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:34:39.638Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"f7b576de-6953-457a-a75b-29e633225f1d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:34:39.638Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"f7b576de-6953-457a-a75b-29e633225f1d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:34:39.638Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"f7b576de-6953-457a-a75b-29e633225f1d","seconds":300}
{"level":"info","ts":"2026-11-03T15:34:41.804Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"6d15f0ac-0a6b-4454-b0a3-6f4c9dab6b97","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:34:41.804Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"6d15f0ac-0a6b-4454-b0a3-6f4c9dab6b97","name":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:34:41.804Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"6d15f0ac-0a6b-4454-b0a3-6f4c9dab6b97","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:34:41.804Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"6d15f0ac-0a6b-4454-b0a3-6f4c9dab6b97"}
{"level":"info","ts":"2026-11-03T15:34:41.804Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"6d15f0ac-0a6b-4454-b0a3-6f4c9dab6b97","RayCluster":"cluster-status-update-mgqjh","State":""}
{"level":"info","ts":"2026-11-03T15:34:41.804Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"6d15f0ac-0a6b-4454-b0a3-6f4c9dab6b97","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:34:42.317Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"e3a53c63-4a41-4404-b00e-e1a95ef39e7f","cluster":"long-running-hvml9"}
{"level":"info","ts":"2026-11-03T15:34:44.805Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"9c6c5955-d9e4-4b04-979b-e6754812430b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:34:44.805Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"9c6c5955-d9e4-4b04-979b-e6754812430b","name":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:34:44.805Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"9c6c5955-d9e4-4b04-979b-e6754812430b","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:34:44.805Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"9c6c5955-d9e4-4b04-979b-e6754812430b"}
{"level":"info","ts":"2026-11-03T15:34:44.805Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"9c6c5955-d9e4-4b04-979b-e6754812430b","RayCluster":"cluster-status-update-mgqjh","State":""}
{"level":"info","ts":"2026-11-03T15:34:44.805Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"9c6c5955-d9e4-4b04-979b-e6754812430b","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:34:47.806Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"01ebeba7-f180-49c3-8a9a-04d8d51190f3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:34:47.806Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"01ebeba7-f180-49c3-8a9a-04d8d51190f3","name":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:34:47.806Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"01ebeba7-f180-49c3-8a9a-04d8d51190f3","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:34:47.806Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"01ebeba7-f180-49c3-8a9a-04d8d51190f3"}
{"level":"info","ts":"2026-11-03T15:34:47.806Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"01ebeba7-f180-49c3-8a9a-04d8d51190f3","RayCluster":"cluster-status-update-mgqjh","State":""}
{"level":"info","ts":"2026-11-03T15:34:47.806Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"01ebeba7-f180-49c3-8a9a-04d8d51190f3","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:34:50.807Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"b65b5e6a-0bc3-496c-bc20-aada7a8e1570","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:34:50.807Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"b65b5e6a-0bc3-496c-bc20-aada7a8e1570","name":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:34:50.807Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"b65b5e6a-0bc3-496c-bc20-aada7a8e1570","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:34:50.807Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"b65b5e6a-0bc3-496c-bc20-aada7a8e1570"}
{"level":"info","ts":"2026-11-03T15:34:50.807Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"b65b5e6a-0bc3-496c-bc20-aada7a8e1570","RayCluster":"cluster-status-update-mgqjh","State":""}
{"level":"info","ts":"2026-11-03T15:34:50.807Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"b65b5e6a-0bc3-496c-bc20-aada7a8e1570","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:34:53.808Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"9357f877-ddf0-459a-b9cd-10e557df3d5c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:34:53.808Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"9357f877-ddf0-459a-b9cd-10e557df3d5c","name":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:34:53.808Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"9357f877-ddf0-459a-b9cd-10e557df3d5c","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:34:53.809Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"9357f877-ddf0-459a-b9cd-10e557df3d5c"}
{"level":"info","ts":"2026-11-03T15:34:53.809Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"9357f877-ddf0-459a-b9cd-10e557df3d5c","RayCluster":"cluster-status-update-mgqjh","State":""}
{"level":"info","ts":"2026-11-03T15:34:53.809Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"9357f877-ddf0-459a-b9cd-10e557df3d5c","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:34:56.809Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"236cbefb-e8cb-43d7-bf7b-e029c785d6e7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:34:56.809Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"236cbefb-e8cb-43d7-bf7b-e029c785d6e7","name":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:34:56.809Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"236cbefb-e8cb-43d7-bf7b-e029c785d6e7","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:34:56.810Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"236cbefb-e8cb-43d7-bf7b-e029c785d6e7"}
{"level":"info","ts":"2026-11-03T15:34:56.810Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"236cbefb-e8cb-43d7-bf7b-e029c785d6e7","RayCluster":"cluster-status-update-mgqjh","State":""}
{"level":"info","ts":"2026-11-03T15:34:56.810Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"236cbefb-e8cb-43d7-bf7b-e029c785d6e7","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:34:59.811Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"bb573970-7f34-4ce5-84f1-c5395539df27","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:34:59.811Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"bb573970-7f34-4ce5-84f1-c5395539df27","name":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:34:59.811Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"bb573970-7f34-4ce5-84f1-c5395539df27","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:34:59.811Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"bb573970-7f34-4ce5-84f1-c5395539df27"}
{"level":"info","ts":"2026-11-03T15:34:59.811Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"bb573970-7f34-4ce5-84f1-c5395539df27","RayCluster":"cluster-status-update-mgqjh","State":""}
{"level":"info","ts":"2026-11-03T15:34:59.811Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"bb573970-7f34-4ce5-84f1-c5395539df27","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:35:00.565Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"6cd49a84-ca8c-4fa5-af82-1a821d529048","cluster":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:35:02.812Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"37d54ebc-20b8-402a-a908-7f8317b37eed","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:35:02.812Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"37d54ebc-20b8-402a-a908-7f8317b37eed","name":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:35:02.812Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"37d54ebc-20b8-402a-a908-7f8317b37eed","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:35:02.812Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"37d54ebc-20b8-402a-a908-7f8317b37eed"}
{"level":"info","ts":"2026-11-03T15:35:02.812Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"37d54ebc-20b8-402a-a908-7f8317b37eed","RayCluster":"cluster-status-update-mgqjh","State":""}
{"level":"info","ts":"2026-11-03T15:35:02.812Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"37d54ebc-20b8-402a-a908-7f8317b37eed","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:35:05.813Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"6d5336d8-fac8-4b10-89f3-52d0088ea76f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:35:05.813Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"6d5336d8-fac8-4b10-89f3-52d0088ea76f","name":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:35:05.813Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"6d5336d8-fac8-4b10-89f3-52d0088ea76f","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:35:05.813Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"6d5336d8-fac8-4b10-89f3-52d0088ea76f"}
{"level":"info","ts":"2026-11-03T15:35:05.813Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"6d5336d8-fac8-4b10-89f3-52d0088ea76f","RayCluster":"cluster-status-update-mgqjh","State":""}
{"level":"info","ts":"2026-11-03T15:35:05.813Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"6d5336d8-fac8-4b10-89f3-52d0088ea76f","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:35:08.814Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"abd6a49a-5448-4f01-bbbc-1f3cb460e317","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:35:08.814Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"abd6a49a-5448-4f01-bbbc-1f3cb460e317","name":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:35:08.814Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"abd6a49a-5448-4f01-bbbc-1f3cb460e317","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:35:08.815Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"abd6a49a-5448-4f01-bbbc-1f3cb460e317"}
{"level":"info","ts":"2026-11-03T15:35:08.815Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"abd6a49a-5448-4f01-bbbc-1f3cb460e317","RayCluster":"cluster-status-update-mgqjh","State":""}
{"level":"info","ts":"2026-11-03T15:35:08.815Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"abd6a49a-5448-4f01-bbbc-1f3cb460e317","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:35:11.815Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"3484f3c7-f08e-417a-a01c-3ad351328176","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:35:11.815Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"3484f3c7-f08e-417a-a01c-3ad351328176","name":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:35:11.815Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"3484f3c7-f08e-417a-a01c-3ad351328176","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:35:11.815Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"3484f3c7-f08e-417a-a01c-3ad351328176"}
{"level":"info","ts":"2026-11-03T15:35:11.815Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"3484f3c7-f08e-417a-a01c-3ad351328176","RayCluster":"cluster-status-update-mgqjh","State":""}
{"level":"info","ts":"2026-11-03T15:35:11.815Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"3484f3c7-f08e-417a-a01c-3ad351328176","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:35:14.816Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"d488b41f-5adc-4c04-9345-ef9bab09d4a7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:35:14.816Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"d488b41f-5adc-4c04-9345-ef9bab09d4a7","name":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:35:14.816Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"d488b41f-5adc-4c04-9345-ef9bab09d4a7","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:35:14.816Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"d488b41f-5adc-4c04-9345-ef9bab09d4a7"}
{"level":"info","ts":"2026-11-03T15:35:14.816Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"d488b41f-5adc-4c04-9345-ef9bab09d4a7","RayCluster":"cluster-status-update-mgqjh","State":""}
{"level":"info","ts":"2026-11-03T15:35:14.816Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"d488b41f-5adc-4c04-9345-ef9bab09d4a7","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:35:17.817Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"72123816-44d7-48dc-9eb1-f54422ac5471","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:35:17.817Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"72123816-44d7-48dc-9eb1-f54422ac5471","name":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:35:17.817Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"72123816-44d7-48dc-9eb1-f54422ac5471","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:35:17.817Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"72123816-44d7-48dc-9eb1-f54422ac5471"}
{"level":"info","ts":"2026-11-03T15:35:17.817Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"72123816-44d7-48dc-9eb1-f54422ac5471","RayCluster":"cluster-status-update-mgqjh","State":""}
{"level":"info","ts":"2026-11-03T15:35:17.817Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"72123816-44d7-48dc-9eb1-f54422ac5471","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:35:20.817Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"ac6dcb63-a3a2-4e26-991c-2250b970daf8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:35:20.817Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"ac6dcb63-a3a2-4e26-991c-2250b970daf8","name":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:35:20.817Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"ac6dcb63-a3a2-4e26-991c-2250b970daf8","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:35:20.818Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"ac6dcb63-a3a2-4e26-991c-2250b970daf8"}
{"level":"info","ts":"2026-11-03T15:35:20.818Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"ac6dcb63-a3a2-4e26-991c-2250b970daf8","RayCluster":"cluster-status-update-mgqjh","State":""}
{"level":"info","ts":"2026-11-03T15:35:20.818Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"ac6dcb63-a3a2-4e26-991c-2250b970daf8","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:35:23.818Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"c8b11060-c9aa-414c-a35e-4493467968c3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:35:23.818Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"c8b11060-c9aa-414c-a35e-4493467968c3","name":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:35:23.818Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"c8b11060-c9aa-414c-a35e-4493467968c3","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:35:23.818Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"c8b11060-c9aa-414c-a35e-4493467968c3"}
{"level":"info","ts":"2026-11-03T15:35:23.818Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"c8b11060-c9aa-414c-a35e-4493467968c3","RayCluster":"cluster-status-update-mgqjh","State":""}
{"level":"info","ts":"2026-11-03T15:35:23.818Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"c8b11060-c9aa-414c-a35e-4493467968c3","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:35:26.820Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"e7f69079-2d2e-4cbf-98d4-2c12538a4c77","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:35:26.820Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"e7f69079-2d2e-4cbf-98d4-2c12538a4c77","name":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:35:26.820Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"e7f69079-2d2e-4cbf-98d4-2c12538a4c77","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:35:26.820Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"e7f69079-2d2e-4cbf-98d4-2c12538a4c77"}
{"level":"info","ts":"2026-11-03T15:35:26.820Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"e7f69079-2d2e-4cbf-98d4-2c12538a4c77","RayCluster":"cluster-status-update-mgqjh","State":""}
{"level":"info","ts":"2026-11-03T15:35:26.820Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"e7f69079-2d2e-4cbf-98d4-2c12538a4c77","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:35:29.821Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"193a1ffa-c453-4dac-af98-8337fe4c4e4e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:35:29.821Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"193a1ffa-c453-4dac-af98-8337fe4c4e4e","name":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:35:29.821Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"193a1ffa-c453-4dac-af98-8337fe4c4e4e","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:35:29.821Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"193a1ffa-c453-4dac-af98-8337fe4c4e4e"}
{"level":"info","ts":"2026-11-03T15:35:29.821Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"193a1ffa-c453-4dac-af98-8337fe4c4e4e","RayCluster":"cluster-status-update-mgqjh","State":""}
{"level":"info","ts":"2026-11-03T15:35:29.821Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"193a1ffa-c453-4dac-af98-8337fe4c4e4e","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:35:32.822Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"21285897-a0a7-4064-af7a-72f8a5da48a2","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:35:32.822Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"21285897-a0a7-4064-af7a-72f8a5da48a2","name":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:35:32.822Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"21285897-a0a7-4064-af7a-72f8a5da48a2","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:35:32.822Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"21285897-a0a7-4064-af7a-72f8a5da48a2"}
{"level":"info","ts":"2026-11-03T15:35:32.822Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"21285897-a0a7-4064-af7a-72f8a5da48a2","RayCluster":"cluster-status-update-mgqjh","State":""}
{"level":"info","ts":"2026-11-03T15:35:32.822Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"21285897-a0a7-4064-af7a-72f8a5da48a2","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:35:35.823Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"ed6c83c6-929e-4080-b928-957d729de578","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:35:35.823Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"ed6c83c6-929e-4080-b928-957d729de578","name":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:35:35.823Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"ed6c83c6-929e-4080-b928-957d729de578","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:35:35.823Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"ed6c83c6-929e-4080-b928-957d729de578"}
{"level":"info","ts":"2026-11-03T15:35:35.823Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"ed6c83c6-929e-4080-b928-957d729de578","RayCluster":"cluster-status-update-mgqjh","State":""}
{"level":"info","ts":"2026-11-03T15:35:35.823Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"ed6c83c6-929e-4080-b928-957d729de578","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:35:38.824Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"01395ac3-a38b-46ce-bfb3-dd48bbe65822","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:35:38.824Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"01395ac3-a38b-46ce-bfb3-dd48bbe65822","name":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:35:38.824Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"01395ac3-a38b-46ce-bfb3-dd48bbe65822","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:35:38.824Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"01395ac3-a38b-46ce-bfb3-dd48bbe65822"}
{"level":"info","ts":"2026-11-03T15:35:38.824Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"01395ac3-a38b-46ce-bfb3-dd48bbe65822","RayCluster":"cluster-status-update-mgqjh","State":""}
{"level":"info","ts":"2026-11-03T15:35:38.824Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"01395ac3-a38b-46ce-bfb3-dd48bbe65822","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:35:41.825Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"9fd78ccc-9912-4e6c-a82b-7fbd7e3e7c02","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:35:41.825Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"9fd78ccc-9912-4e6c-a82b-7fbd7e3e7c02","name":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:35:41.825Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"9fd78ccc-9912-4e6c-a82b-7fbd7e3e7c02","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:35:41.825Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"9fd78ccc-9912-4e6c-a82b-7fbd7e3e7c02"}
{"level":"info","ts":"2026-11-03T15:35:41.825Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"9fd78ccc-9912-4e6c-a82b-7fbd7e3e7c02","RayCluster":"cluster-status-update-mgqjh","State":""}
{"level":"info","ts":"2026-11-03T15:35:41.825Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"9fd78ccc-9912-4e6c-a82b-7fbd7e3e7c02","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:35:43.127Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"7f8bf4fc-5c2a-4970-8987-c68cf54d0011","cluster":"long-running-hvml9"}
{"level":"info","ts":"2026-11-03T15:35:44.827Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"5fec11a3-6332-4bfc-9093-69384cbb169d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:35:44.827Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"5fec11a3-6332-4bfc-9093-69384cbb169d","name":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:35:44.827Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"5fec11a3-6332-4bfc-9093-69384cbb169d","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:35:44.827Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"5fec11a3-6332-4bfc-9093-69384cbb169d"}
{"level":"info","ts":"2026-11-03T15:35:44.827Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"5fec11a3-6332-4bfc-9093-69384cbb169d","RayCluster":"cluster-status-update-mgqjh","State":""}
{"level":"info","ts":"2026-11-03T15:35:44.827Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"5fec11a3-6332-4bfc-9093-69384cbb169d","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:35:47.827Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"a6bed8f2-ec63-494a-b0c0-f8405fdb3d52","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:35:47.827Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"a6bed8f2-ec63-494a-b0c0-f8405fdb3d52","name":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:35:47.827Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"a6bed8f2-ec63-494a-b0c0-f8405fdb3d52","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:35:47.827Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"a6bed8f2-ec63-494a-b0c0-f8405fdb3d52"}
{"level":"info","ts":"2026-11-03T15:35:47.828Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"a6bed8f2-ec63-494a-b0c0-f8405fdb3d52","RayCluster":"cluster-status-update-mgqjh","State":""}
{"level":"info","ts":"2026-11-03T15:35:47.828Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"a6bed8f2-ec63-494a-b0c0-f8405fdb3d52","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:35:50.289Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"c1c2464c-f2ff-4556-869e-ffa8ad0a7d73"}
{"level":"info","ts":"2026-11-03T15:35:50.289Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"c1c2464c-f2ff-4556-869e-ffa8ad0a7d73"}
{"level":"info","ts":"2026-11-03T15:35:50.290Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"c1c2464c-f2ff-4556-869e-ffa8ad0a7d73","1 head service found":"cluster-status-update-mgqjh-head-svc"}
{"level":"info","ts":"2026-11-03T15:35:50.290Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"c1c2464c-f2ff-4556-869e-ffa8ad0a7d73","Found 1 head Pod":"cluster-status-update-mgqjh-head-5wcnb","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:35:50.290Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"c1c2464c-f2ff-4556-869e-ffa8ad0a7d73","head Pod":"cluster-status-update-mgqjh-head-5wcnb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod cluster-status-update-mgqjh-head-5wcnb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:35:50.290Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"c1c2464c-f2ff-4556-869e-ffa8ad0a7d73","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:35:50.290Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"c1c2464c-f2ff-4556-869e-ffa8ad0a7d73","worker Pod":"cluster-status-update-mgqjh-small-group-worker-d5k5g","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod cluster-status-update-mgqjh-small-group-worker-d5k5g. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:35:50.290Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"c1c2464c-f2ff-4556-869e-ffa8ad0a7d73","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:35:50.290Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"c1c2464c-f2ff-4556-869e-ffa8ad0a7d73","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:35:50.290Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"c1c2464c-f2ff-4556-869e-ffa8ad0a7d73","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:35:50.290Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"c1c2464c-f2ff-4556-869e-ffa8ad0a7d73","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:35:50.290Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"c1c2464c-f2ff-4556-869e-ffa8ad0a7d73","podName":"cluster-status-update-mgqjh-head-5wcnb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:35:50.290Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"c1c2464c-f2ff-4556-869e-ffa8ad0a7d73","podName":"cluster-status-update-mgqjh-head-5wcnb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:35:50.290Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"c1c2464c-f2ff-4556-869e-ffa8ad0a7d73","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:35:50.290Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"c1c2464c-f2ff-4556-869e-ffa8ad0a7d73","seconds":300}
{"level":"info","ts":"2026-11-03T15:35:50.829Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"ef172996-b2f3-4b27-8bb0-94aeb66b824f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:35:50.829Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"ef172996-b2f3-4b27-8bb0-94aeb66b824f","name":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:35:50.829Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"ef172996-b2f3-4b27-8bb0-94aeb66b824f","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:35:50.829Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"ef172996-b2f3-4b27-8bb0-94aeb66b824f"}
{"level":"info","ts":"2026-11-03T15:35:50.829Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"ef172996-b2f3-4b27-8bb0-94aeb66b824f","RayCluster":"cluster-status-update-mgqjh","State":""}
{"level":"info","ts":"2026-11-03T15:35:50.829Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"ef172996-b2f3-4b27-8bb0-94aeb66b824f","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:35:51.295Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"ed80941a-679e-40fb-b242-d3e410f0dfee"}
{"level":"info","ts":"2026-11-03T15:35:51.295Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"ed80941a-679e-40fb-b242-d3e410f0dfee"}
{"level":"info","ts":"2026-11-03T15:35:51.295Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"ed80941a-679e-40fb-b242-d3e410f0dfee","1 head service found":"cluster-status-update-mgqjh-head-svc"}
{"level":"info","ts":"2026-11-03T15:35:51.295Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"ed80941a-679e-40fb-b242-d3e410f0dfee","Found 1 head Pod":"cluster-status-update-mgqjh-head-5wcnb","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:35:51.295Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"ed80941a-679e-40fb-b242-d3e410f0dfee","head Pod":"cluster-status-update-mgqjh-head-5wcnb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod cluster-status-update-mgqjh-head-5wcnb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:35:51.295Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"ed80941a-679e-40fb-b242-d3e410f0dfee","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:35:51.295Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"ed80941a-679e-40fb-b242-d3e410f0dfee","worker Pod":"cluster-status-update-mgqjh-small-group-worker-d5k5g","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod cluster-status-update-mgqjh-small-group-worker-d5k5g. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:35:51.295Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"ed80941a-679e-40fb-b242-d3e410f0dfee","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:35:51.295Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"ed80941a-679e-40fb-b242-d3e410f0dfee","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:35:51.295Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"ed80941a-679e-40fb-b242-d3e410f0dfee","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:35:51.295Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"ed80941a-679e-40fb-b242-d3e410f0dfee","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:35:51.295Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"ed80941a-679e-40fb-b242-d3e410f0dfee","podName":"cluster-status-update-mgqjh-head-5wcnb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:35:51.295Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"ed80941a-679e-40fb-b242-d3e410f0dfee","podName":"cluster-status-update-mgqjh-head-5wcnb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:35:51.295Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"ed80941a-679e-40fb-b242-d3e410f0dfee","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:35:51.295Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"ed80941a-679e-40fb-b242-d3e410f0dfee","seconds":300}
{"level":"info","ts":"2026-11-03T15:35:53.830Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"315a44f1-ae86-4374-9ef6-0ed6ec8116fa","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:35:53.830Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"315a44f1-ae86-4374-9ef6-0ed6ec8116fa","name":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:35:53.830Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"315a44f1-ae86-4374-9ef6-0ed6ec8116fa","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:35:53.830Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"315a44f1-ae86-4374-9ef6-0ed6ec8116fa"}
{"level":"info","ts":"2026-11-03T15:35:53.830Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"315a44f1-ae86-4374-9ef6-0ed6ec8116fa","RayCluster":"cluster-status-update-mgqjh","State":""}
{"level":"info","ts":"2026-11-03T15:35:53.830Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"315a44f1-ae86-4374-9ef6-0ed6ec8116fa","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:35:56.831Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"bdbaceb1-5d10-4ffc-ba4f-69a45eacb87a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:35:56.831Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"bdbaceb1-5d10-4ffc-ba4f-69a45eacb87a","name":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:35:56.831Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"bdbaceb1-5d10-4ffc-ba4f-69a45eacb87a","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:35:56.831Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"bdbaceb1-5d10-4ffc-ba4f-69a45eacb87a"}
{"level":"info","ts":"2026-11-03T15:35:56.831Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"bdbaceb1-5d10-4ffc-ba4f-69a45eacb87a","RayCluster":"cluster-status-update-mgqjh","State":""}
{"level":"info","ts":"2026-11-03T15:35:56.831Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"bdbaceb1-5d10-4ffc-ba4f-69a45eacb87a","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:35:59.832Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"b5fb4ef4-145b-4a67-b280-51da9bbe1c80","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:35:59.832Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"b5fb4ef4-145b-4a67-b280-51da9bbe1c80","name":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:35:59.832Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"b5fb4ef4-145b-4a67-b280-51da9bbe1c80","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:35:59.832Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"b5fb4ef4-145b-4a67-b280-51da9bbe1c80"}
{"level":"info","ts":"2026-11-03T15:35:59.832Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"b5fb4ef4-145b-4a67-b280-51da9bbe1c80","RayCluster":"cluster-status-update-mgqjh","State":""}
{"level":"info","ts":"2026-11-03T15:35:59.832Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"b5fb4ef4-145b-4a67-b280-51da9bbe1c80","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:36:00.138Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"8158a565-796c-43c8-ac76-8c7b65034c65"}
{"level":"info","ts":"2026-11-03T15:36:00.138Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"8158a565-796c-43c8-ac76-8c7b65034c65"}
{"level":"info","ts":"2026-11-03T15:36:00.138Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"8158a565-796c-43c8-ac76-8c7b65034c65","1 head service found":"cluster-status-update-mgqjh-head-svc"}
{"level":"info","ts":"2026-11-03T15:36:00.139Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"8158a565-796c-43c8-ac76-8c7b65034c65","Found 1 head Pod":"cluster-status-update-mgqjh-head-5wcnb","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:36:00.139Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"8158a565-796c-43c8-ac76-8c7b65034c65","head Pod":"cluster-status-update-mgqjh-head-5wcnb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod cluster-status-update-mgqjh-head-5wcnb. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:36:00.139Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"8158a565-796c-43c8-ac76-8c7b65034c65","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:36:00.139Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"8158a565-796c-43c8-ac76-8c7b65034c65","worker Pod":"cluster-status-update-mgqjh-small-group-worker-d5k5g","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod cluster-status-update-mgqjh-small-group-worker-d5k5g. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:36:00.139Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"8158a565-796c-43c8-ac76-8c7b65034c65","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:36:00.139Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"8158a565-796c-43c8-ac76-8c7b65034c65","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:36:00.139Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"8158a565-796c-43c8-ac76-8c7b65034c65","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:36:00.139Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"8158a565-796c-43c8-ac76-8c7b65034c65","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:36:00.139Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"8158a565-796c-43c8-ac76-8c7b65034c65","podName":"cluster-status-update-mgqjh-head-5wcnb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:36:00.139Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"8158a565-796c-43c8-ac76-8c7b65034c65","podName":"cluster-status-update-mgqjh-head-5wcnb","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:36:00.139Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"8158a565-796c-43c8-ac76-8c7b65034c65","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:36:00.139Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"8158a565-796c-43c8-ac76-8c7b65034c65","seconds":300}
{"level":"info","ts":"2026-11-03T15:36:01.099Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"73122350-b940-4839-9e4a-e4ef28a10af2"}
{"level":"info","ts":"2026-11-03T15:36:01.099Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"73122350-b940-4839-9e4a-e4ef28a10af2"}
{"level":"info","ts":"2026-11-03T15:36:01.099Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"73122350-b940-4839-9e4a-e4ef28a10af2","1 head service found":"cluster-status-update-mgqjh-head-svc"}
{"level":"info","ts":"2026-11-03T15:36:01.099Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"73122350-b940-4839-9e4a-e4ef28a10af2","Found 1 head Pod":"cluster-status-update-mgqjh-head-5wcnb","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:36:01.099Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"73122350-b940-4839-9e4a-e4ef28a10af2","head Pod":"cluster-status-update-mgqjh-head-5wcnb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod cluster-status-update-mgqjh-head-5wcnb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:36:01.099Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"73122350-b940-4839-9e4a-e4ef28a10af2","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:36:01.099Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"73122350-b940-4839-9e4a-e4ef28a10af2","worker Pod":"cluster-status-update-mgqjh-small-group-worker-d5k5g","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod cluster-status-update-mgqjh-small-group-worker-d5k5g. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:36:01.099Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"73122350-b940-4839-9e4a-e4ef28a10af2","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:36:01.099Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"73122350-b940-4839-9e4a-e4ef28a10af2","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:36:01.099Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"73122350-b940-4839-9e4a-e4ef28a10af2","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:36:01.099Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"73122350-b940-4839-9e4a-e4ef28a10af2","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:36:01.099Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"73122350-b940-4839-9e4a-e4ef28a10af2","podName":"cluster-status-update-mgqjh-head-5wcnb","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:34:28Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:36:01.099Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"73122350-b940-4839-9e4a-e4ef28a10af2","podName":"cluster-status-update-mgqjh-head-5wcnb","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:34:28Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:36:01.099Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"73122350-b940-4839-9e4a-e4ef28a10af2","name":"cluster-status-update-mgqjh","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:01Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:36:01.161Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"a6bd033e-6993-4fdd-9815-300b1e900306","cluster":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:36:01.161Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"d567036e-6ad8-40c7-832f-7a582bd54493","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:36:01.161Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"d567036e-6ad8-40c7-832f-7a582bd54493","name":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:36:01.161Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"d567036e-6ad8-40c7-832f-7a582bd54493","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:36:01.161Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"d567036e-6ad8-40c7-832f-7a582bd54493"}
{"level":"info","ts":"2026-11-03T15:36:01.161Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"d567036e-6ad8-40c7-832f-7a582bd54493","RayCluster":"cluster-status-update-mgqjh","State":""}
{"level":"info","ts":"2026-11-03T15:36:01.161Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"d567036e-6ad8-40c7-832f-7a582bd54493","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:34:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:01Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:36:01.161Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"d567036e-6ad8-40c7-832f-7a582bd54493","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:36:01.223Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"42e2a05b-42f1-4e4e-9a93-088b45ee7f7f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:36:01.223Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"42e2a05b-42f1-4e4e-9a93-088b45ee7f7f","name":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:36:01.223Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"42e2a05b-42f1-4e4e-9a93-088b45ee7f7f","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:36:01.223Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"42e2a05b-42f1-4e4e-9a93-088b45ee7f7f"}
{"level":"info","ts":"2026-11-03T15:36:01.223Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"42e2a05b-42f1-4e4e-9a93-088b45ee7f7f","RayCluster":"cluster-status-update-mgqjh","State":""}
{"level":"info","ts":"2026-11-03T15:36:01.223Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"42e2a05b-42f1-4e4e-9a93-088b45ee7f7f","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:01Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:01Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:36:03.162Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"695342f3-7581-446f-9745-1945cdc7be1d"}
{"level":"info","ts":"2026-11-03T15:36:03.162Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"695342f3-7581-446f-9745-1945cdc7be1d"}
{"level":"info","ts":"2026-11-03T15:36:03.162Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"695342f3-7581-446f-9745-1945cdc7be1d","1 head service found":"cluster-status-update-mgqjh-head-svc"}
{"level":"info","ts":"2026-11-03T15:36:03.162Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"695342f3-7581-446f-9745-1945cdc7be1d","Found 1 head Pod":"cluster-status-update-mgqjh-head-5wcnb","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:36:03.162Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"695342f3-7581-446f-9745-1945cdc7be1d","head Pod":"cluster-status-update-mgqjh-head-5wcnb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod cluster-status-update-mgqjh-head-5wcnb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:36:03.162Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"695342f3-7581-446f-9745-1945cdc7be1d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:36:03.162Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"695342f3-7581-446f-9745-1945cdc7be1d","worker Pod":"cluster-status-update-mgqjh-small-group-worker-d5k5g","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod cluster-status-update-mgqjh-small-group-worker-d5k5g. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:36:03.162Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"695342f3-7581-446f-9745-1945cdc7be1d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:36:03.162Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"695342f3-7581-446f-9745-1945cdc7be1d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:36:03.162Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"695342f3-7581-446f-9745-1945cdc7be1d","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:36:03.162Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"695342f3-7581-446f-9745-1945cdc7be1d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:36:03.162Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"695342f3-7581-446f-9745-1945cdc7be1d","podName":"cluster-status-update-mgqjh-head-5wcnb","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:34:28Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:36:03.162Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"695342f3-7581-446f-9745-1945cdc7be1d","podName":"cluster-status-update-mgqjh-head-5wcnb","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:34:28Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:36:03.162Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"695342f3-7581-446f-9745-1945cdc7be1d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:36:03.162Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"695342f3-7581-446f-9745-1945cdc7be1d","seconds":300}
{"level":"info","ts":"2026-11-03T15:36:04.224Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"a5c698bd-3bd9-424c-bb1f-bed432426921","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:36:04.225Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"a5c698bd-3bd9-424c-bb1f-bed432426921","name":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:36:04.225Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"a5c698bd-3bd9-424c-bb1f-bed432426921","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:36:04.225Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"a5c698bd-3bd9-424c-bb1f-bed432426921"}
{"level":"info","ts":"2026-11-03T15:36:04.225Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"a5c698bd-3bd9-424c-bb1f-bed432426921","RayCluster":"cluster-status-update-mgqjh","State":""}
{"level":"info","ts":"2026-11-03T15:36:04.225Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"a5c698bd-3bd9-424c-bb1f-bed432426921","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:01Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:01Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:36:04.351Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"31077f14-720c-46a3-99a8-7afe13e393db"}
{"level":"info","ts":"2026-11-03T15:36:04.351Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"31077f14-720c-46a3-99a8-7afe13e393db"}
{"level":"info","ts":"2026-11-03T15:36:04.351Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"31077f14-720c-46a3-99a8-7afe13e393db","1 head service found":"cluster-status-update-mgqjh-head-svc"}
{"level":"info","ts":"2026-11-03T15:36:04.351Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"31077f14-720c-46a3-99a8-7afe13e393db","Found 1 head Pod":"cluster-status-update-mgqjh-head-5wcnb","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:36:04.351Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"31077f14-720c-46a3-99a8-7afe13e393db","head Pod":"cluster-status-update-mgqjh-head-5wcnb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod cluster-status-update-mgqjh-head-5wcnb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:36:04.351Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"31077f14-720c-46a3-99a8-7afe13e393db","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:36:04.351Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"31077f14-720c-46a3-99a8-7afe13e393db","worker Pod":"cluster-status-update-mgqjh-small-group-worker-d5k5g","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod cluster-status-update-mgqjh-small-group-worker-d5k5g. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:36:04.351Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"31077f14-720c-46a3-99a8-7afe13e393db","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:36:04.351Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"31077f14-720c-46a3-99a8-7afe13e393db","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:36:04.351Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"31077f14-720c-46a3-99a8-7afe13e393db","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:36:04.351Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"31077f14-720c-46a3-99a8-7afe13e393db","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:36:04.351Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"31077f14-720c-46a3-99a8-7afe13e393db","podName":"cluster-status-update-mgqjh-head-5wcnb","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:34:28Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:36:04.351Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"31077f14-720c-46a3-99a8-7afe13e393db","podName":"cluster-status-update-mgqjh-head-5wcnb","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:34:28Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:36:04.352Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"31077f14-720c-46a3-99a8-7afe13e393db","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:36:04.352Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"31077f14-720c-46a3-99a8-7afe13e393db","seconds":300}
{"level":"info","ts":"2026-11-03T15:36:05.412Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"2271dbba-2f6f-4758-88f8-a7cf67ae31a7"}
{"level":"info","ts":"2026-11-03T15:36:05.412Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"2271dbba-2f6f-4758-88f8-a7cf67ae31a7"}
{"level":"info","ts":"2026-11-03T15:36:05.412Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"2271dbba-2f6f-4758-88f8-a7cf67ae31a7","1 head service found":"cluster-status-update-mgqjh-head-svc"}
{"level":"info","ts":"2026-11-03T15:36:05.412Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"2271dbba-2f6f-4758-88f8-a7cf67ae31a7","Found 1 head Pod":"cluster-status-update-mgqjh-head-5wcnb","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:36:05.412Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"2271dbba-2f6f-4758-88f8-a7cf67ae31a7","head Pod":"cluster-status-update-mgqjh-head-5wcnb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod cluster-status-update-mgqjh-head-5wcnb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:36:05.412Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"2271dbba-2f6f-4758-88f8-a7cf67ae31a7","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:36:05.412Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"2271dbba-2f6f-4758-88f8-a7cf67ae31a7","worker Pod":"cluster-status-update-mgqjh-small-group-worker-d5k5g","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod cluster-status-update-mgqjh-small-group-worker-d5k5g. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:36:05.412Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"2271dbba-2f6f-4758-88f8-a7cf67ae31a7","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:36:05.412Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"2271dbba-2f6f-4758-88f8-a7cf67ae31a7","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:36:05.413Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"2271dbba-2f6f-4758-88f8-a7cf67ae31a7","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:36:05.413Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"2271dbba-2f6f-4758-88f8-a7cf67ae31a7","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:36:05.413Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"2271dbba-2f6f-4758-88f8-a7cf67ae31a7","podName":"cluster-status-update-mgqjh-head-5wcnb","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:34:28Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:36:05.413Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"2271dbba-2f6f-4758-88f8-a7cf67ae31a7","podName":"cluster-status-update-mgqjh-head-5wcnb","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:34:28Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:36:05.413Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"2271dbba-2f6f-4758-88f8-a7cf67ae31a7","name":"cluster-status-update-mgqjh","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:01Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:36:05.473Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"5c85526b-7e02-4500-9a9a-55542c659982","cluster":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:36:05.474Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"37517882-d28b-49aa-bf3f-b19f7c5fb04d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:36:05.474Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"37517882-d28b-49aa-bf3f-b19f7c5fb04d","name":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:36:05.474Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"37517882-d28b-49aa-bf3f-b19f7c5fb04d","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:36:05.474Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"37517882-d28b-49aa-bf3f-b19f7c5fb04d"}
{"level":"info","ts":"2026-11-03T15:36:05.474Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"37517882-d28b-49aa-bf3f-b19f7c5fb04d","RayCluster":"cluster-status-update-mgqjh","State":""}
{"level":"info","ts":"2026-11-03T15:36:05.474Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"37517882-d28b-49aa-bf3f-b19f7c5fb04d","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:01Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:36:05.474Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"37517882-d28b-49aa-bf3f-b19f7c5fb04d","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:36:05.535Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"eaa1e0df-235a-49bc-8035-167f02775b49","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:36:05.535Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"eaa1e0df-235a-49bc-8035-167f02775b49","name":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:36:05.535Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"eaa1e0df-235a-49bc-8035-167f02775b49","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:36:05.536Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"eaa1e0df-235a-49bc-8035-167f02775b49"}
{"level":"info","ts":"2026-11-03T15:36:05.536Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"eaa1e0df-235a-49bc-8035-167f02775b49","RayCluster":"cluster-status-update-mgqjh","State":""}
{"level":"info","ts":"2026-11-03T15:36:05.536Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"eaa1e0df-235a-49bc-8035-167f02775b49","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:36:07.474Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"7a16f7c0-b063-46c4-98b9-bc0ffa110059"}
{"level":"info","ts":"2026-11-03T15:36:07.474Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"7a16f7c0-b063-46c4-98b9-bc0ffa110059"}
{"level":"info","ts":"2026-11-03T15:36:07.474Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"7a16f7c0-b063-46c4-98b9-bc0ffa110059","1 head service found":"cluster-status-update-mgqjh-head-svc"}
{"level":"info","ts":"2026-11-03T15:36:07.474Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"7a16f7c0-b063-46c4-98b9-bc0ffa110059","Found 1 head Pod":"cluster-status-update-mgqjh-head-5wcnb","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:36:07.474Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"7a16f7c0-b063-46c4-98b9-bc0ffa110059","head Pod":"cluster-status-update-mgqjh-head-5wcnb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod cluster-status-update-mgqjh-head-5wcnb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:36:07.474Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"7a16f7c0-b063-46c4-98b9-bc0ffa110059","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:36:07.474Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"7a16f7c0-b063-46c4-98b9-bc0ffa110059","worker Pod":"cluster-status-update-mgqjh-small-group-worker-d5k5g","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod cluster-status-update-mgqjh-small-group-worker-d5k5g. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:36:07.474Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"7a16f7c0-b063-46c4-98b9-bc0ffa110059","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:36:07.474Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"7a16f7c0-b063-46c4-98b9-bc0ffa110059","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:36:07.474Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"7a16f7c0-b063-46c4-98b9-bc0ffa110059","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:36:07.474Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"7a16f7c0-b063-46c4-98b9-bc0ffa110059","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:36:07.474Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"7a16f7c0-b063-46c4-98b9-bc0ffa110059","podName":"cluster-status-update-mgqjh-head-5wcnb","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:34:28Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:36:07.474Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"7a16f7c0-b063-46c4-98b9-bc0ffa110059","podName":"cluster-status-update-mgqjh-head-5wcnb","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:34:28Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:36:07.474Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"7a16f7c0-b063-46c4-98b9-bc0ffa110059","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:36:07.474Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"7a16f7c0-b063-46c4-98b9-bc0ffa110059","seconds":300}
{"level":"info","ts":"2026-11-03T15:36:08.536Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"b1beb123-8f9c-4121-993d-ca43dce1564c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:36:08.536Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"b1beb123-8f9c-4121-993d-ca43dce1564c","name":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:36:08.536Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"b1beb123-8f9c-4121-993d-ca43dce1564c","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:36:08.536Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"b1beb123-8f9c-4121-993d-ca43dce1564c"}
{"level":"info","ts":"2026-11-03T15:36:08.536Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"b1beb123-8f9c-4121-993d-ca43dce1564c","RayCluster":"cluster-status-update-mgqjh","State":""}
{"level":"info","ts":"2026-11-03T15:36:08.536Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"b1beb123-8f9c-4121-993d-ca43dce1564c","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:36:11.537Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"a51fbc47-8e3c-40d3-9b11-d64e82dd6378","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:36:11.537Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"a51fbc47-8e3c-40d3-9b11-d64e82dd6378","name":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:36:11.537Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"a51fbc47-8e3c-40d3-9b11-d64e82dd6378","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:36:11.538Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"a51fbc47-8e3c-40d3-9b11-d64e82dd6378"}
{"level":"info","ts":"2026-11-03T15:36:11.538Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"a51fbc47-8e3c-40d3-9b11-d64e82dd6378","RayCluster":"cluster-status-update-mgqjh","State":""}
{"level":"info","ts":"2026-11-03T15:36:11.538Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"a51fbc47-8e3c-40d3-9b11-d64e82dd6378","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:36:12.155Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"5861328c-b4f6-44d8-af21-5724ce3ce149"}
{"level":"info","ts":"2026-11-03T15:36:12.155Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"5861328c-b4f6-44d8-af21-5724ce3ce149"}
{"level":"info","ts":"2026-11-03T15:36:12.155Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"5861328c-b4f6-44d8-af21-5724ce3ce149","1 head service found":"cluster-status-update-mgqjh-head-svc"}
{"level":"info","ts":"2026-11-03T15:36:12.155Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"5861328c-b4f6-44d8-af21-5724ce3ce149","Found 1 head Pod":"cluster-status-update-mgqjh-head-5wcnb","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:36:12.155Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"5861328c-b4f6-44d8-af21-5724ce3ce149","head Pod":"cluster-status-update-mgqjh-head-5wcnb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod cluster-status-update-mgqjh-head-5wcnb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:36:12.155Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"5861328c-b4f6-44d8-af21-5724ce3ce149","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:36:12.155Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"5861328c-b4f6-44d8-af21-5724ce3ce149","worker Pod":"cluster-status-update-mgqjh-small-group-worker-d5k5g","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod cluster-status-update-mgqjh-small-group-worker-d5k5g. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:36:12.155Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"5861328c-b4f6-44d8-af21-5724ce3ce149","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:36:12.155Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"5861328c-b4f6-44d8-af21-5724ce3ce149","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:36:12.155Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"5861328c-b4f6-44d8-af21-5724ce3ce149","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:36:12.155Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"5861328c-b4f6-44d8-af21-5724ce3ce149","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:36:12.156Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"5861328c-b4f6-44d8-af21-5724ce3ce149","podName":"cluster-status-update-mgqjh-small-group-worker-d5k5g","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:34:28Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:36:12.156Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"5861328c-b4f6-44d8-af21-5724ce3ce149","podName":"cluster-status-update-mgqjh-small-group-worker-d5k5g","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:34:28Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:36:12.156Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"5861328c-b4f6-44d8-af21-5724ce3ce149","name":"cluster-status-update-mgqjh","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:12Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:36:12Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:36:12.217Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"1bdccd6c-0135-4c83-8483-48015856b3a0","cluster":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:36:12.218Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"f3330b45-7d2c-436f-8019-3ba5aee1ed15","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:36:12.218Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"f3330b45-7d2c-436f-8019-3ba5aee1ed15","name":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:36:12.218Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"f3330b45-7d2c-436f-8019-3ba5aee1ed15","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:36:12.218Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"f3330b45-7d2c-436f-8019-3ba5aee1ed15"}
{"level":"info","ts":"2026-11-03T15:36:12.218Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"f3330b45-7d2c-436f-8019-3ba5aee1ed15","RayCluster":"cluster-status-update-mgqjh","State":""}
{"level":"info","ts":"2026-11-03T15:36:12.218Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"f3330b45-7d2c-436f-8019-3ba5aee1ed15","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:05Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:12Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:36:12Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:36:12.218Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"f3330b45-7d2c-436f-8019-3ba5aee1ed15","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:36:12.279Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"159653ba-8fe3-43da-8e8a-125d858241c5","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:36:12.279Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"159653ba-8fe3-43da-8e8a-125d858241c5","name":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:36:12.279Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"159653ba-8fe3-43da-8e8a-125d858241c5","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:36:12.279Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"159653ba-8fe3-43da-8e8a-125d858241c5"}
{"level":"info","ts":"2026-11-03T15:36:12.279Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"159653ba-8fe3-43da-8e8a-125d858241c5","RayCluster":"cluster-status-update-mgqjh","State":""}
{"level":"info","ts":"2026-11-03T15:36:12.279Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"159653ba-8fe3-43da-8e8a-125d858241c5","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:12Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:36:12Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:12Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:36:12Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:36:14.218Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"690d8e11-5eb2-42f7-a708-429e0dbce014"}
{"level":"info","ts":"2026-11-03T15:36:14.218Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"690d8e11-5eb2-42f7-a708-429e0dbce014"}
{"level":"info","ts":"2026-11-03T15:36:14.218Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"690d8e11-5eb2-42f7-a708-429e0dbce014","1 head service found":"cluster-status-update-mgqjh-head-svc"}
{"level":"info","ts":"2026-11-03T15:36:14.218Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"690d8e11-5eb2-42f7-a708-429e0dbce014","Found 1 head Pod":"cluster-status-update-mgqjh-head-5wcnb","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:36:14.218Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"690d8e11-5eb2-42f7-a708-429e0dbce014","head Pod":"cluster-status-update-mgqjh-head-5wcnb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod cluster-status-update-mgqjh-head-5wcnb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:36:14.218Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"690d8e11-5eb2-42f7-a708-429e0dbce014","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:36:14.218Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"690d8e11-5eb2-42f7-a708-429e0dbce014","worker Pod":"cluster-status-update-mgqjh-small-group-worker-d5k5g","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod cluster-status-update-mgqjh-small-group-worker-d5k5g. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:36:14.218Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"690d8e11-5eb2-42f7-a708-429e0dbce014","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:36:14.218Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"690d8e11-5eb2-42f7-a708-429e0dbce014","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:36:14.218Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"690d8e11-5eb2-42f7-a708-429e0dbce014","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:36:14.218Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"690d8e11-5eb2-42f7-a708-429e0dbce014","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:36:14.219Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"690d8e11-5eb2-42f7-a708-429e0dbce014","podName":"cluster-status-update-mgqjh-small-group-worker-d5k5g","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:34:28Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:36:14.219Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"690d8e11-5eb2-42f7-a708-429e0dbce014","podName":"cluster-status-update-mgqjh-small-group-worker-d5k5g","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:34:28Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:36:14.219Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"690d8e11-5eb2-42f7-a708-429e0dbce014","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:36:14.219Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"690d8e11-5eb2-42f7-a708-429e0dbce014","seconds":300}
{"level":"info","ts":"2026-11-03T15:36:15.280Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"ae1f2abd-a75b-419b-b28a-be75d9d21e1d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:36:15.280Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"ae1f2abd-a75b-419b-b28a-be75d9d21e1d","name":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:36:15.280Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"ae1f2abd-a75b-419b-b28a-be75d9d21e1d","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:36:15.280Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"ae1f2abd-a75b-419b-b28a-be75d9d21e1d"}
{"level":"info","ts":"2026-11-03T15:36:15.280Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"ae1f2abd-a75b-419b-b28a-be75d9d21e1d","RayCluster":"cluster-status-update-mgqjh","State":""}
{"level":"info","ts":"2026-11-03T15:36:15.280Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"ae1f2abd-a75b-419b-b28a-be75d9d21e1d","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:12Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:36:12Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:12Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:36:12Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:36:16.420Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"9e9dbb4a-d0fb-4351-a4e2-2b0515e1d047"}
{"level":"info","ts":"2026-11-03T15:36:16.420Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"9e9dbb4a-d0fb-4351-a4e2-2b0515e1d047"}
{"level":"info","ts":"2026-11-03T15:36:16.420Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"9e9dbb4a-d0fb-4351-a4e2-2b0515e1d047","1 head service found":"cluster-status-update-mgqjh-head-svc"}
{"level":"info","ts":"2026-11-03T15:36:16.420Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"9e9dbb4a-d0fb-4351-a4e2-2b0515e1d047","Found 1 head Pod":"cluster-status-update-mgqjh-head-5wcnb","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:36:16.420Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"9e9dbb4a-d0fb-4351-a4e2-2b0515e1d047","head Pod":"cluster-status-update-mgqjh-head-5wcnb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod cluster-status-update-mgqjh-head-5wcnb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:36:16.420Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"9e9dbb4a-d0fb-4351-a4e2-2b0515e1d047","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:36:16.420Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"9e9dbb4a-d0fb-4351-a4e2-2b0515e1d047","worker Pod":"cluster-status-update-mgqjh-small-group-worker-d5k5g","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod cluster-status-update-mgqjh-small-group-worker-d5k5g. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:36:16.420Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"9e9dbb4a-d0fb-4351-a4e2-2b0515e1d047","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:36:16.420Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"9e9dbb4a-d0fb-4351-a4e2-2b0515e1d047","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:36:16.420Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"9e9dbb4a-d0fb-4351-a4e2-2b0515e1d047","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:36:16.420Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"9e9dbb4a-d0fb-4351-a4e2-2b0515e1d047","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:36:16.420Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"9e9dbb4a-d0fb-4351-a4e2-2b0515e1d047","name":"cluster-status-update-mgqjh","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:12Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:36:12Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:16Z","stateTransitionTimes":{"ready":"2026-11-03T15:36:16Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:36:12Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:36:16Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:36:16.481Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"ed8009ff-74c9-4de4-b78f-faeacd186fc7","cluster":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:36:16.481Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"df7366fd-13db-4dd2-99fd-0f43a8199a40","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:36:16.481Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"df7366fd-13db-4dd2-99fd-0f43a8199a40","name":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:36:16.482Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"df7366fd-13db-4dd2-99fd-0f43a8199a40","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:36:16.482Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"df7366fd-13db-4dd2-99fd-0f43a8199a40"}
{"level":"info","ts":"2026-11-03T15:36:16.544Z","logger":"controllers.RayJob","msg":"Created submitter Kubernetes Job for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"df7366fd-13db-4dd2-99fd-0f43a8199a40","Kubernetes Job":"cluster-status-update"}
{"level":"info","ts":"2026-11-03T15:36:16.544Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"df7366fd-13db-4dd2-99fd-0f43a8199a40","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:12Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:36:12Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","dashboardURL":"cluster-status-update-mgqjh-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:12Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:36:12Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:36:16.544Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"df7366fd-13db-4dd2-99fd-0f43a8199a40","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-11-03T15:36:16.605Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"9845ffc2-cc1f-448e-b9ec-e5d26a51a26e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:36:16.605Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"9845ffc2-cc1f-448e-b9ec-e5d26a51a26e","name":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:36:16.605Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"9845ffc2-cc1f-448e-b9ec-e5d26a51a26e","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:36:16.605Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"9845ffc2-cc1f-448e-b9ec-e5d26a51a26e"}
{"level":"info","ts":"2026-11-03T15:36:16.605Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job for RayJob already exists","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"9845ffc2-cc1f-448e-b9ec-e5d26a51a26e","Kubernetes Job":"cluster-status-update"}
{"level":"info","ts":"2026-11-03T15:36:16.605Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"9845ffc2-cc1f-448e-b9ec-e5d26a51a26e","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:12Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:36:12Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","dashboardURL":"cluster-status-update-mgqjh-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:12Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:36:12Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:36:16.605Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"9845ffc2-cc1f-448e-b9ec-e5d26a51a26e","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-11-03T15:36:16.660Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"9845ffc2-cc1f-448e-b9ec-e5d26a51a26e","error":"Operation cannot be fulfilled on rayjobs.ray.io \"cluster-status-update\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-11-03T15:36:16.660Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"9845ffc2-cc1f-448e-b9ec-e5d26a51a26e"}
{"level":"error","ts":"2026-11-03T15:36:16.660Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"9845ffc2-cc1f-448e-b9ec-e5d26a51a26e","error":"Operation cannot be fulfilled on rayjobs.ray.io \"cluster-status-update\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:36:16.660Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"09e0d226-041d-4346-b4e8-1f7ed53da243","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:36:16.660Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"09e0d226-041d-4346-b4e8-1f7ed53da243","name":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:36:16.660Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"09e0d226-041d-4346-b4e8-1f7ed53da243","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:36:16.660Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"09e0d226-041d-4346-b4e8-1f7ed53da243"}
{"level":"info","ts":"2026-11-03T15:36:16.660Z","logger":"controllers.RayJob","msg":"Create a Kubernetes Service","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"09e0d226-041d-4346-b4e8-1f7ed53da243","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:36:16.724Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"498c98bb-4012-4253-b309-9486ddb69a80","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:36:16.724Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"498c98bb-4012-4253-b309-9486ddb69a80","name":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:36:16.724Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"498c98bb-4012-4253-b309-9486ddb69a80","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:36:16.724Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"498c98bb-4012-4253-b309-9486ddb69a80"}
{"level":"info","ts":"2026-11-03T15:36:16.724Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"498c98bb-4012-4253-b309-9486ddb69a80","rayCluster":"cluster-status-update-mgqjh","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:36:18.482Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"1014436a-f2e1-40d6-8145-68a3ebd9f8e8"}
{"level":"info","ts":"2026-11-03T15:36:18.482Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"1014436a-f2e1-40d6-8145-68a3ebd9f8e8"}
{"level":"info","ts":"2026-11-03T15:36:18.482Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"1014436a-f2e1-40d6-8145-68a3ebd9f8e8","1 head service found":"cluster-status-update-mgqjh-head-svc"}
{"level":"info","ts":"2026-11-03T15:36:18.483Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"1014436a-f2e1-40d6-8145-68a3ebd9f8e8","Found 1 head Pod":"cluster-status-update-mgqjh-head-5wcnb","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:36:18.483Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"1014436a-f2e1-40d6-8145-68a3ebd9f8e8","head Pod":"cluster-status-update-mgqjh-head-5wcnb","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod cluster-status-update-mgqjh-head-5wcnb. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:36:18.483Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"1014436a-f2e1-40d6-8145-68a3ebd9f8e8","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:36:18.483Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"1014436a-f2e1-40d6-8145-68a3ebd9f8e8","worker Pod":"cluster-status-update-mgqjh-small-group-worker-d5k5g","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod cluster-status-update-mgqjh-small-group-worker-d5k5g. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:36:18.483Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"1014436a-f2e1-40d6-8145-68a3ebd9f8e8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:36:18.483Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"1014436a-f2e1-40d6-8145-68a3ebd9f8e8","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:36:18.483Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"1014436a-f2e1-40d6-8145-68a3ebd9f8e8","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:36:18.483Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"1014436a-f2e1-40d6-8145-68a3ebd9f8e8","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:36:18.483Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"1014436a-f2e1-40d6-8145-68a3ebd9f8e8","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:36:18.483Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"1014436a-f2e1-40d6-8145-68a3ebd9f8e8","seconds":300}
{"level":"info","ts":"2026-11-03T15:36:19.369Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"306a14fc-262a-41c5-8c0a-0d3a5a8ce2b8","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:36:19.369Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"306a14fc-262a-41c5-8c0a-0d3a5a8ce2b8","name":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:36:19.369Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"306a14fc-262a-41c5-8c0a-0d3a5a8ce2b8","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:36:19.370Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"306a14fc-262a-41c5-8c0a-0d3a5a8ce2b8"}
{"level":"info","ts":"2026-11-03T15:36:19.370Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"306a14fc-262a-41c5-8c0a-0d3a5a8ce2b8","rayCluster":"cluster-status-update-mgqjh","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:36:22.376Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"aefc29cc-5aaa-49e8-a957-de50c43e9aee","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:36:22.376Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"aefc29cc-5aaa-49e8-a957-de50c43e9aee","name":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:36:22.376Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"aefc29cc-5aaa-49e8-a957-de50c43e9aee","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:36:22.376Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"aefc29cc-5aaa-49e8-a957-de50c43e9aee"}
{"level":"info","ts":"2026-11-03T15:36:22.377Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"aefc29cc-5aaa-49e8-a957-de50c43e9aee","rayCluster":"cluster-status-update-mgqjh","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:36:25.383Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"19cc7ea9-627a-433b-976e-6fc9013edb45","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:36:25.383Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"19cc7ea9-627a-433b-976e-6fc9013edb45","name":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:36:25.383Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"19cc7ea9-627a-433b-976e-6fc9013edb45","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:36:25.383Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"19cc7ea9-627a-433b-976e-6fc9013edb45"}
{"level":"info","ts":"2026-11-03T15:36:25.383Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"19cc7ea9-627a-433b-976e-6fc9013edb45","rayCluster":"cluster-status-update-mgqjh","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:36:25.389Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"19cc7ea9-627a-433b-976e-6fc9013edb45","oldRayJobStatus":{"rayJobInfo":{},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","dashboardURL":"cluster-status-update-mgqjh-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:12Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:36:12Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:36:25Z"},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","dashboardURL":"cluster-status-update-mgqjh-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:16Z","stateTransitionTimes":{"ready":"2026-11-03T15:36:16Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:36:12Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:36:16Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:36:25.389Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"19cc7ea9-627a-433b-976e-6fc9013edb45","old JobStatus":"","new JobStatus":"PENDING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-11-03T15:36:25.486Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"ee33c690-cecb-4c42-aa36-94b8f0718b9d","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:36:25.486Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"ee33c690-cecb-4c42-aa36-94b8f0718b9d","name":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:36:25.486Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"ee33c690-cecb-4c42-aa36-94b8f0718b9d","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:36:25.487Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"ee33c690-cecb-4c42-aa36-94b8f0718b9d"}
{"level":"info","ts":"2026-11-03T15:36:25.487Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"ee33c690-cecb-4c42-aa36-94b8f0718b9d","rayCluster":"cluster-status-update-mgqjh","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:36:25.490Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"ee33c690-cecb-4c42-aa36-94b8f0718b9d","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:36:25Z"},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","dashboardURL":"cluster-status-update-mgqjh-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:16Z","stateTransitionTimes":{"ready":"2026-11-03T15:36:16Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:36:12Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:36:16Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:36:25Z"},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","dashboardURL":"cluster-status-update-mgqjh-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:16Z","stateTransitionTimes":{"ready":"2026-11-03T15:36:16Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:36:12Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:36:16Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:36:28.491Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"610ae9c3-a44e-41a4-bc2d-3529e435543e","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:36:28.491Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"610ae9c3-a44e-41a4-bc2d-3529e435543e","name":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:36:28.491Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"610ae9c3-a44e-41a4-bc2d-3529e435543e","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:36:28.491Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"610ae9c3-a44e-41a4-bc2d-3529e435543e"}
{"level":"info","ts":"2026-11-03T15:36:28.491Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"610ae9c3-a44e-41a4-bc2d-3529e435543e","rayCluster":"cluster-status-update-mgqjh","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:36:28.494Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"610ae9c3-a44e-41a4-bc2d-3529e435543e","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:36:25Z"},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","dashboardURL":"cluster-status-update-mgqjh-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet.","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:16Z","stateTransitionTimes":{"ready":"2026-11-03T15:36:16Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:36:12Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:36:16Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:36:25Z"},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","dashboardURL":"cluster-status-update-mgqjh-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:16Z","stateTransitionTimes":{"ready":"2026-11-03T15:36:16Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:36:12Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:36:16Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:36:28.494Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"610ae9c3-a44e-41a4-bc2d-3529e435543e","old JobStatus":"PENDING","new JobStatus":"RUNNING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-11-03T15:36:28.603Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"39ce26cb-2b23-4915-b781-4581f151f965","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:36:28.603Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"39ce26cb-2b23-4915-b781-4581f151f965","name":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:36:28.603Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"39ce26cb-2b23-4915-b781-4581f151f965","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:36:28.603Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"39ce26cb-2b23-4915-b781-4581f151f965"}
{"level":"info","ts":"2026-11-03T15:36:28.603Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"39ce26cb-2b23-4915-b781-4581f151f965","rayCluster":"cluster-status-update-mgqjh","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:36:28.606Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"39ce26cb-2b23-4915-b781-4581f151f965","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:36:25Z"},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","dashboardURL":"cluster-status-update-mgqjh-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:16Z","stateTransitionTimes":{"ready":"2026-11-03T15:36:16Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:36:12Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:36:16Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:36:25Z"},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","dashboardURL":"cluster-status-update-mgqjh-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:16Z","stateTransitionTimes":{"ready":"2026-11-03T15:36:16Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:36:12Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:36:16Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:36:28.944Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"f35ecfc4-f7b3-4ec4-bfed-697685134899","cluster":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:36:28.944Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"4d58bcdb-8f15-4031-91eb-255a4810230a","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:36:28.944Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"4d58bcdb-8f15-4031-91eb-255a4810230a","name":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:36:28.944Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"4d58bcdb-8f15-4031-91eb-255a4810230a","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:36:28.944Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"4d58bcdb-8f15-4031-91eb-255a4810230a"}
{"level":"info","ts":"2026-11-03T15:36:28.944Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"4d58bcdb-8f15-4031-91eb-255a4810230a","rayCluster":"cluster-status-update-mgqjh","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:36:28.947Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"4d58bcdb-8f15-4031-91eb-255a4810230a","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:36:25Z"},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","dashboardURL":"cluster-status-update-mgqjh-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:16Z","stateTransitionTimes":{"ready":"2026-11-03T15:36:16Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:36:12Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:36:16Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:36:25Z"},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","dashboardURL":"cluster-status-update-mgqjh-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:16Z","stateTransitionTimes":{"ready":"2026-11-03T15:36:16Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:36:12Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:36:16Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:36:28.947Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"4d58bcdb-8f15-4031-91eb-255a4810230a","old JobStatus":"RUNNING","new JobStatus":"RUNNING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-11-03T15:36:29.042Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"2eb34776-46c7-450c-8de4-9e60710d50a5","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:36:29.042Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"2eb34776-46c7-450c-8de4-9e60710d50a5","name":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:36:29.042Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"2eb34776-46c7-450c-8de4-9e60710d50a5","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:36:29.042Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"2eb34776-46c7-450c-8de4-9e60710d50a5"}
{"level":"info","ts":"2026-11-03T15:36:29.042Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"2eb34776-46c7-450c-8de4-9e60710d50a5","rayCluster":"cluster-status-update-mgqjh","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:36:29.045Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"2eb34776-46c7-450c-8de4-9e60710d50a5","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:36:25Z"},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","dashboardURL":"cluster-status-update-mgqjh-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:16Z","stateTransitionTimes":{"ready":"2026-11-03T15:36:16Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:36:12Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:36:16Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:36:25Z"},"jobId":"cluster-status-update-psdmb","rayClusterName":"cluster-status-update-mgqjh","dashboardURL":"cluster-status-update-mgqjh-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:34:24Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:16Z","stateTransitionTimes":{"ready":"2026-11-03T15:36:16Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.47","serviceIP":"10.132.0.47","podName":"cluster-status-update-mgqjh-head-5wcnb","serviceName":"cluster-status-update-mgqjh-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:36:12Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:36:16Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:34:25Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:34:25Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:36:30.007Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"b216b69f-63f3-4972-89a5-faa92cd82905","DeletionTimestamp":"2026-11-03 15:36:29 +0000 UTC"}
{"level":"info","ts":"2026-11-03T15:36:30.016Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"b216b69f-63f3-4972-89a5-faa92cd82905","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-11-03T15:36:30.082Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"417cb660-cd5b-424c-9ad6-b67e125d531f","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:36:30.082Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"417cb660-cd5b-424c-9ad6-b67e125d531f","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-11-03T15:36:30.116Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"cluster-status-update-mgqjh","reconcileID":"a85ffcae-8c59-4910-aac7-47db05c9dea5","namespacedName":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:36:30.116Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"cluster-status-update-mgqjh","reconcileID":"a85ffcae-8c59-4910-aac7-47db05c9dea5","cluster":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:36:30.116Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"cluster-status-update-mgqjh","reconcileID":"a85ffcae-8c59-4910-aac7-47db05c9dea5","cluster":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:36:30.116Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"cluster-status-update-mgqjh","reconcileID":"a85ffcae-8c59-4910-aac7-47db05c9dea5","namespace":"test-ns-kbx4w","otherAuthClusters":1,"action":"will retain"}
{"level":"info","ts":"2026-11-03T15:36:30.116Z","logger":"authentication-controller","msg":"ReferenceGrant retained (other clusters with auth exist)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"cluster-status-update-mgqjh","reconcileID":"a85ffcae-8c59-4910-aac7-47db05c9dea5","name":"kuberay-gateway-access","namespace":"test-ns-kbx4w","remainingClusters":1}
{"level":"info","ts":"2026-11-03T15:36:30.116Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"reconcileID":"f11f67f0-e08b-483b-8486-a67f3921c77f"}
{"level":"info","ts":"2026-11-03T15:36:30.141Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"417cb660-cd5b-424c-9ad6-b67e125d531f"}
{"level":"info","ts":"2026-11-03T15:36:30.141Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"417cb660-cd5b-424c-9ad6-b67e125d531f","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-11-03T15:36:30.141Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"417cb660-cd5b-424c-9ad6-b67e125d531f","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:36:30.141Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"417cb660-cd5b-424c-9ad6-b67e125d531f","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:36:30.172Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"cluster-status-update-mgqjh","reconcileID":"a85ffcae-8c59-4910-aac7-47db05c9dea5","name":"test-ns-kbx4w-cluster-status-update-mgqjh","namespace":"opendatahub"}
{"level":"info","ts":"2026-11-03T15:36:30.201Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"069d1f84-9458-4239-bb72-d64b857bf746"}
{"level":"info","ts":"2026-11-03T15:36:30.201Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"68aae8ad-cb57-4bdc-a619-f79cd6494a22","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:36:30.201Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"68aae8ad-cb57-4bdc-a619-f79cd6494a22"}
{"level":"info","ts":"2026-11-03T15:36:30.201Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"68aae8ad-cb57-4bdc-a619-f79cd6494a22","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-11-03T15:36:30.201Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"68aae8ad-cb57-4bdc-a619-f79cd6494a22","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-t76gq","rayClusterName":"counter-auth-2v5kp","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:36:30.201Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"68aae8ad-cb57-4bdc-a619-f79cd6494a22","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:36:30.226Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"cluster-status-update-mgqjh","reconcileID":"a85ffcae-8c59-4910-aac7-47db05c9dea5","configMap":"kube-rbac-proxy-config-cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:36:30.257Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"68aae8ad-cb57-4bdc-a619-f79cd6494a22","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter-auth\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-11-03T15:36:30.257Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"68aae8ad-cb57-4bdc-a619-f79cd6494a22"}
{"level":"error","ts":"2026-11-03T15:36:30.257Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"68aae8ad-cb57-4bdc-a619-f79cd6494a22","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter-auth\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:36:30.257Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"138526d3-7969-4621-8c2b-2daf816149d1","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:36:30.257Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"138526d3-7969-4621-8c2b-2daf816149d1","name":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:36:30.257Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"138526d3-7969-4621-8c2b-2daf816149d1","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:36:30.257Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"138526d3-7969-4621-8c2b-2daf816149d1","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:36:30.281Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"cluster-status-update-mgqjh","reconcileID":"a85ffcae-8c59-4910-aac7-47db05c9dea5","serviceAccount":"cluster-status-update-mgqjh-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-11-03T15:36:30.434Z","logger":"raycluster-resource","msg":"default","name":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:36:30.434Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:36:30.465Z","logger":"raycluster-resource","msg":"default","name":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:36:30.466Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:36:30.615Z","logger":"raycluster-resource","msg":"validate create","name":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:36:30.640Z","logger":"raycluster-resource","msg":"validate update","name":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:36:30.668Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"138526d3-7969-4621-8c2b-2daf816149d1","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:36:30.669Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"138526d3-7969-4621-8c2b-2daf816149d1"}
{"level":"info","ts":"2026-11-03T15:36:30.669Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"138526d3-7969-4621-8c2b-2daf816149d1","RayCluster":"counter-auth-qmw8r","State":""}
{"level":"info","ts":"2026-11-03T15:36:30.669Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"138526d3-7969-4621-8c2b-2daf816149d1","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:36:30.669Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"fd6b9541-885b-4757-83db-7949946058a4","cluster":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:36:30.669Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"121ecd48-3274-4509-91e4-d0bc6d003848","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:36:30.669Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"121ecd48-3274-4509-91e4-d0bc6d003848","name":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:36:30.669Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"121ecd48-3274-4509-91e4-d0bc6d003848","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:36:30.669Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"81333430-184f-4297-9646-6d713fc61f74"}
{"level":"info","ts":"2026-11-03T15:36:30.669Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"81333430-184f-4297-9646-6d713fc61f74"}
{"level":"info","ts":"2026-11-03T15:36:30.669Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"121ecd48-3274-4509-91e4-d0bc6d003848"}
{"level":"info","ts":"2026-11-03T15:36:30.669Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"121ecd48-3274-4509-91e4-d0bc6d003848","RayCluster":"counter-auth-qmw8r","State":""}
{"level":"info","ts":"2026-11-03T15:36:30.669Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"121ecd48-3274-4509-91e4-d0bc6d003848","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:36:30.669Z","logger":"controllers.NetworkPolicy","msg":"Reconciling NetworkPolicies for RayCluster","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"fcc72dbd-1351-42e7-bc5b-93c5cc0e0ae4","cluster":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:36:30.695Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"cluster-status-update-mgqjh","reconcileID":"a85ffcae-8c59-4910-aac7-47db05c9dea5","cluster":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:36:30.695Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"cluster-status-update-mgqjh","reconcileID":"eba1b88d-be18-49d7-985b-4300a76378b6","namespacedName":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:36:30.695Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"cluster-status-update-mgqjh","reconcileID":"eba1b88d-be18-49d7-985b-4300a76378b6","cluster":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:36:30.695Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"cluster-status-update-mgqjh","reconcileID":"eba1b88d-be18-49d7-985b-4300a76378b6","cluster":"cluster-status-update-mgqjh"}
{"level":"info","ts":"2026-11-03T15:36:30.695Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"cluster-status-update-mgqjh","reconcileID":"eba1b88d-be18-49d7-985b-4300a76378b6","namespace":"test-ns-kbx4w","otherAuthClusters":2,"action":"will retain"}
{"level":"info","ts":"2026-11-03T15:36:30.695Z","logger":"authentication-controller","msg":"ReferenceGrant retained (other clusters with auth exist)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"cluster-status-update-mgqjh","reconcileID":"eba1b88d-be18-49d7-985b-4300a76378b6","name":"kuberay-gateway-access","namespace":"test-ns-kbx4w","remainingClusters":2}
{"level":"info","ts":"2026-11-03T15:36:30.695Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"cluster-status-update","namespace":"test-ns-kbx4w"},"reconcileID":"15a79f61-b620-4b70-aa5e-ea104346d052"}
{"level":"info","ts":"2026-11-03T15:36:30.743Z","logger":"controllers.NetworkPolicy","msg":"Successfully created NetworkPolicy","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"fcc72dbd-1351-42e7-bc5b-93c5cc0e0ae4","name":"counter-auth-qmw8r-head"}
{"level":"error","ts":"2026-11-03T15:36:30.766Z","logger":"authentication-controller","msg":"Failed to remove finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"cluster-status-update-mgqjh","reconcileID":"eba1b88d-be18-49d7-985b-4300a76378b6","error":"rayclusters.ray.io \"cluster-status-update-mgqjh\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleDeletion\n\t/workspace/controllers/ray/authentication_controller.go:195\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:36:30.766Z","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"cluster-status-update-mgqjh","reconcileID":"eba1b88d-be18-49d7-985b-4300a76378b6"}
{"level":"error","ts":"2026-11-03T15:36:30.766Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"cluster-status-update-mgqjh","reconcileID":"eba1b88d-be18-49d7-985b-4300a76378b6","error":"rayclusters.ray.io \"cluster-status-update-mgqjh\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:36:30.766Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"counter-auth-qmw8r","reconcileID":"06f20395-3773-4afa-8747-e0f4ce874c99","namespacedName":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:36:30.766Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"counter-auth-qmw8r","reconcileID":"06f20395-3773-4afa-8747-e0f4ce874c99","mode":"IntegratedOAuth","cluster":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:36:30.766Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"counter-auth-qmw8r","reconcileID":"06f20395-3773-4afa-8747-e0f4ce874c99","cluster":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:36:30.766Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"counter-auth-qmw8r","reconcileID":"06f20395-3773-4afa-8747-e0f4ce874c99","cluster":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:36:30.797Z","logger":"controllers.NetworkPolicy","msg":"Successfully created NetworkPolicy","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"fcc72dbd-1351-42e7-bc5b-93c5cc0e0ae4","name":"counter-auth-qmw8r-workers-small-group"}
{"level":"info","ts":"2026-11-03T15:36:30.797Z","logger":"controllers.NetworkPolicy","msg":"Successfully reconciled NetworkPolicies for RayCluster","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"fcc72dbd-1351-42e7-bc5b-93c5cc0e0ae4","cluster":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:36:30.797Z","logger":"controllers.NetworkPolicy","msg":"Reconciling NetworkPolicies for RayCluster","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"4a1b0750-bef9-4f9b-8c96-845641020ccd","cluster":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:36:30.797Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"81333430-184f-4297-9646-6d713fc61f74","name":"counter-auth-qmw8r-head-svc"}
{"level":"info","ts":"2026-11-03T15:36:30.797Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"81333430-184f-4297-9646-6d713fc61f74"}
{"level":"info","ts":"2026-11-03T15:36:30.824Z","logger":"raycluster-resource","msg":"default","name":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:36:30.824Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:36:30.847Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"81333430-184f-4297-9646-6d713fc61f74","error":"certificate ray-head-cert-counter-auth-qmw8r not found: certificates.cert-manager.io \"ray-head-cert-counter-auth-qmw8r\" not found"}
{"level":"error","ts":"2026-11-03T15:36:30.847Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"81333430-184f-4297-9646-6d713fc61f74","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-counter-auth-qmw8r not found: certificates.cert-manager.io \"ray-head-cert-counter-auth-qmw8r\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:36:30.847Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"81333430-184f-4297-9646-6d713fc61f74","error":"mTLS secrets not ready: certificate ray-head-cert-counter-auth-qmw8r not found: certificates.cert-manager.io \"ray-head-cert-counter-auth-qmw8r\" not found"}
{"level":"info","ts":"2026-11-03T15:36:30.847Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"81333430-184f-4297-9646-6d713fc61f74","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"counter-auth-qmw8r","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:36:30.847Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"81333430-184f-4297-9646-6d713fc61f74","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"counter-auth-qmw8r","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:36:30.847Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"81333430-184f-4297-9646-6d713fc61f74","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"counter-auth-qmw8r","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:36:30.847Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"81333430-184f-4297-9646-6d713fc61f74","name":"counter-auth-qmw8r","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:36:30.879Z","logger":"raycluster-resource","msg":"validate update","name":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:36:30.907Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"81333430-184f-4297-9646-6d713fc61f74"}
{"level":"error","ts":"2026-11-03T15:36:30.907Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"81333430-184f-4297-9646-6d713fc61f74","error":"mTLS secrets not ready: certificate ray-head-cert-counter-auth-qmw8r not found: certificates.cert-manager.io \"ray-head-cert-counter-auth-qmw8r\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:36:30.907Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"8a40d69c-e56b-40b8-a2c8-367013586c01"}
{"level":"info","ts":"2026-11-03T15:36:30.907Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"8a40d69c-e56b-40b8-a2c8-367013586c01"}
{"level":"info","ts":"2026-11-03T15:36:30.907Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"8a40d69c-e56b-40b8-a2c8-367013586c01","1 head service found":"counter-auth-qmw8r-head-svc"}
{"level":"info","ts":"2026-11-03T15:36:30.907Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"29d90364-04b4-43fc-adec-b6caf4c960de","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:36:30.907Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"29d90364-04b4-43fc-adec-b6caf4c960de","name":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:36:30.907Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"29d90364-04b4-43fc-adec-b6caf4c960de","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:36:30.907Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"8a40d69c-e56b-40b8-a2c8-367013586c01"}
{"level":"info","ts":"2026-11-03T15:36:30.907Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"29d90364-04b4-43fc-adec-b6caf4c960de"}
{"level":"info","ts":"2026-11-03T15:36:30.907Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"29d90364-04b4-43fc-adec-b6caf4c960de","RayCluster":"counter-auth-qmw8r","State":""}
{"level":"info","ts":"2026-11-03T15:36:30.907Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"29d90364-04b4-43fc-adec-b6caf4c960de","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:36:30.907Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"29d90364-04b4-43fc-adec-b6caf4c960de","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:36:30.913Z","logger":"controllers.NetworkPolicy","msg":"Successfully reconciled NetworkPolicies for RayCluster","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"4a1b0750-bef9-4f9b-8c96-845641020ccd","cluster":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:36:30.942Z","logger":"raycluster-resource","msg":"default","name":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:36:30.942Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:36:30.956Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"8a40d69c-e56b-40b8-a2c8-367013586c01","error":"certificate ray-head-cert-counter-auth-qmw8r not found: certificates.cert-manager.io \"ray-head-cert-counter-auth-qmw8r\" not found"}
{"level":"error","ts":"2026-11-03T15:36:30.956Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"8a40d69c-e56b-40b8-a2c8-367013586c01","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-counter-auth-qmw8r not found: certificates.cert-manager.io \"ray-head-cert-counter-auth-qmw8r\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:36:30.957Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"8a40d69c-e56b-40b8-a2c8-367013586c01","error":"mTLS secrets not ready: certificate ray-head-cert-counter-auth-qmw8r not found: certificates.cert-manager.io \"ray-head-cert-counter-auth-qmw8r\" not found"}
{"level":"info","ts":"2026-11-03T15:36:30.957Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"8a40d69c-e56b-40b8-a2c8-367013586c01","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"counter-auth-qmw8r","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:36:30.957Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"8a40d69c-e56b-40b8-a2c8-367013586c01","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"counter-auth-qmw8r","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:36:30.957Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"8a40d69c-e56b-40b8-a2c8-367013586c01","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"counter-auth-qmw8r","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:36:30.957Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"8a40d69c-e56b-40b8-a2c8-367013586c01"}
{"level":"error","ts":"2026-11-03T15:36:30.957Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"8a40d69c-e56b-40b8-a2c8-367013586c01","error":"mTLS secrets not ready: certificate ray-head-cert-counter-auth-qmw8r not found: certificates.cert-manager.io \"ray-head-cert-counter-auth-qmw8r\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:36:30.967Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"6622d423-96f9-469d-a135-7f94a73ad675"}
{"level":"info","ts":"2026-11-03T15:36:30.967Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"6622d423-96f9-469d-a135-7f94a73ad675"}
{"level":"info","ts":"2026-11-03T15:36:30.967Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"6622d423-96f9-469d-a135-7f94a73ad675","1 head service found":"counter-auth-qmw8r-head-svc"}
{"level":"info","ts":"2026-11-03T15:36:30.967Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"6622d423-96f9-469d-a135-7f94a73ad675"}
{"level":"info","ts":"2026-11-03T15:36:30.968Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"a3d3e03f-eafc-4263-8540-2f1d790a5003","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:36:30.968Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"a3d3e03f-eafc-4263-8540-2f1d790a5003","name":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:36:30.968Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"a3d3e03f-eafc-4263-8540-2f1d790a5003","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:36:30.968Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"a3d3e03f-eafc-4263-8540-2f1d790a5003"}
{"level":"info","ts":"2026-11-03T15:36:30.968Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"a3d3e03f-eafc-4263-8540-2f1d790a5003","RayCluster":"counter-auth-qmw8r","State":""}
{"level":"info","ts":"2026-11-03T15:36:30.968Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"a3d3e03f-eafc-4263-8540-2f1d790a5003","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"error","ts":"2026-11-03T15:36:30.991Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"counter-auth-qmw8r","reconcileID":"06f20395-3773-4afa-8747-e0f4ce874c99","error":"failed to add finalizer: Operation cannot be fulfilled on rayclusters.ray.io \"counter-auth-qmw8r\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:167\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:36:30.991Z","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"counter-auth-qmw8r","reconcileID":"06f20395-3773-4afa-8747-e0f4ce874c99"}
{"level":"error","ts":"2026-11-03T15:36:30.991Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"counter-auth-qmw8r","reconcileID":"06f20395-3773-4afa-8747-e0f4ce874c99","error":"failed to add finalizer: Operation cannot be fulfilled on rayclusters.ray.io \"counter-auth-qmw8r\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:36:30.991Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"cluster-status-update-mgqjh","reconcileID":"f4568551-58f7-4f62-ae88-452667272adf","namespacedName":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:36:30.991Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"cluster-status-update-mgqjh","reconcileID":"f4568551-58f7-4f62-ae88-452667272adf"}
{"level":"info","ts":"2026-11-03T15:36:30.991Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"cluster-status-update-mgqjh","reconcileID":"f4568551-58f7-4f62-ae88-452667272adf","namespace":"test-ns-kbx4w","authClusters":2}
{"level":"info","ts":"2026-11-03T15:36:30.991Z","logger":"authentication-controller","msg":"ReferenceGrant not orphaned (clusters with auth exist)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"cluster-status-update-mgqjh","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"cluster-status-update-mgqjh","reconcileID":"f4568551-58f7-4f62-ae88-452667272adf","namespace":"test-ns-kbx4w","authClusters":2}
{"level":"info","ts":"2026-11-03T15:36:30.991Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"counter-auth-qmw8r","reconcileID":"f847cb6c-520f-4834-bd40-1433ab331a1e","namespacedName":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:36:30.991Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"counter-auth-qmw8r","reconcileID":"f847cb6c-520f-4834-bd40-1433ab331a1e","mode":"IntegratedOAuth","cluster":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:36:30.991Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"counter-auth-qmw8r","reconcileID":"f847cb6c-520f-4834-bd40-1433ab331a1e","cluster":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:36:30.991Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"counter-auth-qmw8r","reconcileID":"f847cb6c-520f-4834-bd40-1433ab331a1e","cluster":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:36:31.016Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"6622d423-96f9-469d-a135-7f94a73ad675","error":"certificate ray-head-cert-counter-auth-qmw8r not found: certificates.cert-manager.io \"ray-head-cert-counter-auth-qmw8r\" not found"}
{"level":"error","ts":"2026-11-03T15:36:31.016Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"6622d423-96f9-469d-a135-7f94a73ad675","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-counter-auth-qmw8r not found: certificates.cert-manager.io \"ray-head-cert-counter-auth-qmw8r\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:36:31.016Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"6622d423-96f9-469d-a135-7f94a73ad675","error":"mTLS secrets not ready: certificate ray-head-cert-counter-auth-qmw8r not found: certificates.cert-manager.io \"ray-head-cert-counter-auth-qmw8r\" not found"}
{"level":"info","ts":"2026-11-03T15:36:31.017Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"6622d423-96f9-469d-a135-7f94a73ad675","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"counter-auth-qmw8r","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:36:31.017Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"6622d423-96f9-469d-a135-7f94a73ad675","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"counter-auth-qmw8r","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:36:31.017Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"6622d423-96f9-469d-a135-7f94a73ad675","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"counter-auth-qmw8r","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:36:31.017Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"6622d423-96f9-469d-a135-7f94a73ad675"}
{"level":"error","ts":"2026-11-03T15:36:31.017Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"6622d423-96f9-469d-a135-7f94a73ad675","error":"mTLS secrets not ready: certificate ray-head-cert-counter-auth-qmw8r not found: certificates.cert-manager.io \"ray-head-cert-counter-auth-qmw8r\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:36:31.038Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"f8ebde38-fb6c-4ef9-b954-458c73d9f92a"}
{"level":"info","ts":"2026-11-03T15:36:31.038Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"f8ebde38-fb6c-4ef9-b954-458c73d9f92a"}
{"level":"info","ts":"2026-11-03T15:36:31.038Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"f8ebde38-fb6c-4ef9-b954-458c73d9f92a","1 head service found":"counter-auth-qmw8r-head-svc"}
{"level":"info","ts":"2026-11-03T15:36:31.038Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"f8ebde38-fb6c-4ef9-b954-458c73d9f92a"}
{"level":"info","ts":"2026-11-03T15:36:31.059Z","logger":"raycluster-resource","msg":"default","name":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:36:31.059Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:36:31.089Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"f8ebde38-fb6c-4ef9-b954-458c73d9f92a","error":"certificate ray-head-cert-counter-auth-qmw8r not found: certificates.cert-manager.io \"ray-head-cert-counter-auth-qmw8r\" not found"}
{"level":"error","ts":"2026-11-03T15:36:31.089Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"f8ebde38-fb6c-4ef9-b954-458c73d9f92a","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-counter-auth-qmw8r not found: certificates.cert-manager.io \"ray-head-cert-counter-auth-qmw8r\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:36:31.089Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"f8ebde38-fb6c-4ef9-b954-458c73d9f92a","error":"mTLS secrets not ready: certificate ray-head-cert-counter-auth-qmw8r not found: certificates.cert-manager.io \"ray-head-cert-counter-auth-qmw8r\" not found"}
{"level":"info","ts":"2026-11-03T15:36:31.089Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"f8ebde38-fb6c-4ef9-b954-458c73d9f92a","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"counter-auth-qmw8r","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:36:31.089Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"f8ebde38-fb6c-4ef9-b954-458c73d9f92a","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"counter-auth-qmw8r","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:36:31.089Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"f8ebde38-fb6c-4ef9-b954-458c73d9f92a","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"counter-auth-qmw8r","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:36:31.090Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"f8ebde38-fb6c-4ef9-b954-458c73d9f92a"}
{"level":"error","ts":"2026-11-03T15:36:31.090Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"f8ebde38-fb6c-4ef9-b954-458c73d9f92a","error":"mTLS secrets not ready: certificate ray-head-cert-counter-auth-qmw8r not found: certificates.cert-manager.io \"ray-head-cert-counter-auth-qmw8r\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:36:31.115Z","logger":"raycluster-resource","msg":"validate update","name":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:36:31.130Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"50cbdd1e-ef3b-454a-bd79-d025f03804e9"}
{"level":"info","ts":"2026-11-03T15:36:31.130Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"50cbdd1e-ef3b-454a-bd79-d025f03804e9"}
{"level":"info","ts":"2026-11-03T15:36:31.130Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"50cbdd1e-ef3b-454a-bd79-d025f03804e9","1 head service found":"counter-auth-qmw8r-head-svc"}
{"level":"info","ts":"2026-11-03T15:36:31.130Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"50cbdd1e-ef3b-454a-bd79-d025f03804e9"}
{"level":"info","ts":"2026-11-03T15:36:31.172Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"8acebc3a-e4ac-4611-81e9-731fc88da6f9","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:36:31.172Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"8acebc3a-e4ac-4611-81e9-731fc88da6f9","name":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:36:31.172Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"8acebc3a-e4ac-4611-81e9-731fc88da6f9","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:36:31.172Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"8acebc3a-e4ac-4611-81e9-731fc88da6f9"}
{"level":"info","ts":"2026-11-03T15:36:31.172Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"8acebc3a-e4ac-4611-81e9-731fc88da6f9","RayCluster":"counter-auth-qmw8r","State":""}
{"level":"info","ts":"2026-11-03T15:36:31.172Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"8acebc3a-e4ac-4611-81e9-731fc88da6f9","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:36:31.180Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"50cbdd1e-ef3b-454a-bd79-d025f03804e9","error":"certificate ray-head-cert-counter-auth-qmw8r not found: certificates.cert-manager.io \"ray-head-cert-counter-auth-qmw8r\" not found"}
{"level":"error","ts":"2026-11-03T15:36:31.180Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"50cbdd1e-ef3b-454a-bd79-d025f03804e9","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-counter-auth-qmw8r not found: certificates.cert-manager.io \"ray-head-cert-counter-auth-qmw8r\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:36:31.180Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"50cbdd1e-ef3b-454a-bd79-d025f03804e9","error":"mTLS secrets not ready: certificate ray-head-cert-counter-auth-qmw8r not found: certificates.cert-manager.io \"ray-head-cert-counter-auth-qmw8r\" not found"}
{"level":"info","ts":"2026-11-03T15:36:31.180Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"50cbdd1e-ef3b-454a-bd79-d025f03804e9","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"counter-auth-qmw8r","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:36:31.180Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"50cbdd1e-ef3b-454a-bd79-d025f03804e9","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"counter-auth-qmw8r","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:36:31.180Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"50cbdd1e-ef3b-454a-bd79-d025f03804e9","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"counter-auth-qmw8r","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:36:31.180Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"50cbdd1e-ef3b-454a-bd79-d025f03804e9"}
{"level":"error","ts":"2026-11-03T15:36:31.180Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"50cbdd1e-ef3b-454a-bd79-d025f03804e9","error":"mTLS secrets not ready: certificate ray-head-cert-counter-auth-qmw8r not found: certificates.cert-manager.io \"ray-head-cert-counter-auth-qmw8r\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:36:31.223Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"counter-auth-qmw8r","reconcileID":"f847cb6c-520f-4834-bd40-1433ab331a1e","name":"counter-auth-qmw8r-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-11-03T15:36:31.260Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"73d74c6a-64e4-42ec-9f1d-a66e3be22885"}
{"level":"info","ts":"2026-11-03T15:36:31.260Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"73d74c6a-64e4-42ec-9f1d-a66e3be22885"}
{"level":"info","ts":"2026-11-03T15:36:31.260Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"73d74c6a-64e4-42ec-9f1d-a66e3be22885","1 head service found":"counter-auth-qmw8r-head-svc"}
{"level":"info","ts":"2026-11-03T15:36:31.260Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"73d74c6a-64e4-42ec-9f1d-a66e3be22885"}
{"level":"info","ts":"2026-11-03T15:36:31.277Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"counter-auth-qmw8r","reconcileID":"f847cb6c-520f-4834-bd40-1433ab331a1e","name":"test-ns-kbx4w-counter-auth-qmw8r","operation":"created"}
{"level":"info","ts":"2026-11-03T15:36:31.313Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"73d74c6a-64e4-42ec-9f1d-a66e3be22885","error":"certificate ray-head-cert-counter-auth-qmw8r not found: certificates.cert-manager.io \"ray-head-cert-counter-auth-qmw8r\" not found"}
{"level":"error","ts":"2026-11-03T15:36:31.313Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"73d74c6a-64e4-42ec-9f1d-a66e3be22885","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-counter-auth-qmw8r not found: certificates.cert-manager.io \"ray-head-cert-counter-auth-qmw8r\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:36:31.313Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"73d74c6a-64e4-42ec-9f1d-a66e3be22885","error":"mTLS secrets not ready: certificate ray-head-cert-counter-auth-qmw8r not found: certificates.cert-manager.io \"ray-head-cert-counter-auth-qmw8r\" not found"}
{"level":"info","ts":"2026-11-03T15:36:31.313Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"73d74c6a-64e4-42ec-9f1d-a66e3be22885","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"counter-auth-qmw8r","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:36:31.314Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"73d74c6a-64e4-42ec-9f1d-a66e3be22885","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"counter-auth-qmw8r","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:36:31.314Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"73d74c6a-64e4-42ec-9f1d-a66e3be22885","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"counter-auth-qmw8r","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:36:31.314Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"73d74c6a-64e4-42ec-9f1d-a66e3be22885"}
{"level":"error","ts":"2026-11-03T15:36:31.314Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"73d74c6a-64e4-42ec-9f1d-a66e3be22885","error":"mTLS secrets not ready: certificate ray-head-cert-counter-auth-qmw8r not found: certificates.cert-manager.io \"ray-head-cert-counter-auth-qmw8r\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:36:31.329Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"counter-auth-qmw8r","reconcileID":"f847cb6c-520f-4834-bd40-1433ab331a1e","name":"kube-rbac-proxy-config-counter-auth-qmw8r","operation":"created"}
{"level":"error","ts":"2026-11-03T15:36:31.383Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"counter-auth-qmw8r","reconcileID":"f847cb6c-520f-4834-bd40-1433ab331a1e","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"counter-auth-qmw8r\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:167\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:36:31.384Z","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"counter-auth-qmw8r","reconcileID":"f847cb6c-520f-4834-bd40-1433ab331a1e"}
{"level":"error","ts":"2026-11-03T15:36:31.384Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"counter-auth-qmw8r","reconcileID":"f847cb6c-520f-4834-bd40-1433ab331a1e","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"counter-auth-qmw8r\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:36:31.384Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"counter-auth-qmw8r","reconcileID":"afe20245-4299-4521-a2e8-9c9009779064","namespacedName":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:36:31.384Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"counter-auth-qmw8r","reconcileID":"afe20245-4299-4521-a2e8-9c9009779064","mode":"IntegratedOAuth","cluster":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:36:31.384Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"counter-auth-qmw8r","reconcileID":"afe20245-4299-4521-a2e8-9c9009779064","cluster":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:36:31.445Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"counter-auth-qmw8r","reconcileID":"afe20245-4299-4521-a2e8-9c9009779064","cluster":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:36:31.445Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"1e38803a-a6bd-4c57-af30-c0c14bc4cd9a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:36:31.445Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"1e38803a-a6bd-4c57-af30-c0c14bc4cd9a","name":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:36:31.445Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"1e38803a-a6bd-4c57-af30-c0c14bc4cd9a","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:36:31.446Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"1e38803a-a6bd-4c57-af30-c0c14bc4cd9a"}
{"level":"info","ts":"2026-11-03T15:36:31.446Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"1e38803a-a6bd-4c57-af30-c0c14bc4cd9a","RayCluster":"counter-auth-qmw8r","State":""}
{"level":"info","ts":"2026-11-03T15:36:31.446Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"1e38803a-a6bd-4c57-af30-c0c14bc4cd9a","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:36:31.446Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"1e38803a-a6bd-4c57-af30-c0c14bc4cd9a","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:36:31.475Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"e9f723d3-01e3-4616-93f3-740f308f94e4"}
{"level":"info","ts":"2026-11-03T15:36:31.475Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"e9f723d3-01e3-4616-93f3-740f308f94e4"}
{"level":"info","ts":"2026-11-03T15:36:31.475Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"e9f723d3-01e3-4616-93f3-740f308f94e4","1 head service found":"counter-auth-qmw8r-head-svc"}
{"level":"info","ts":"2026-11-03T15:36:31.475Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"e9f723d3-01e3-4616-93f3-740f308f94e4"}
{"level":"info","ts":"2026-11-03T15:36:31.506Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"afedfc50-aa23-450f-a36d-7c2c57a6fc55","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:36:31.506Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"afedfc50-aa23-450f-a36d-7c2c57a6fc55","name":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:36:31.506Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"afedfc50-aa23-450f-a36d-7c2c57a6fc55","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:36:31.506Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"afedfc50-aa23-450f-a36d-7c2c57a6fc55"}
{"level":"info","ts":"2026-11-03T15:36:31.506Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"afedfc50-aa23-450f-a36d-7c2c57a6fc55","RayCluster":"counter-auth-qmw8r","State":""}
{"level":"info","ts":"2026-11-03T15:36:31.506Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"afedfc50-aa23-450f-a36d-7c2c57a6fc55","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:36:31.525Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"e9f723d3-01e3-4616-93f3-740f308f94e4","error":"certificate ray-head-cert-counter-auth-qmw8r is not ready at current generation 1"}
{"level":"error","ts":"2026-11-03T15:36:31.525Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"e9f723d3-01e3-4616-93f3-740f308f94e4","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-counter-auth-qmw8r is not ready at current generation 1","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:36:31.525Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"e9f723d3-01e3-4616-93f3-740f308f94e4","error":"mTLS secrets not ready: certificate ray-head-cert-counter-auth-qmw8r is not ready at current generation 1"}
{"level":"info","ts":"2026-11-03T15:36:31.525Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"e9f723d3-01e3-4616-93f3-740f308f94e4","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"counter-auth-qmw8r","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:36:31.525Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"e9f723d3-01e3-4616-93f3-740f308f94e4","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"counter-auth-qmw8r","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:36:31.525Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"e9f723d3-01e3-4616-93f3-740f308f94e4","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"counter-auth-qmw8r","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:36:31.525Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"e9f723d3-01e3-4616-93f3-740f308f94e4"}
{"level":"error","ts":"2026-11-03T15:36:31.525Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"e9f723d3-01e3-4616-93f3-740f308f94e4","error":"mTLS secrets not ready: certificate ray-head-cert-counter-auth-qmw8r is not ready at current generation 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:36:31.659Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"fd6b9541-885b-4757-83db-7949946058a4","certificate":"ray-head-cert-counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:36:31.659Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"fd6b9541-885b-4757-83db-7949946058a4"}
{"level":"info","ts":"2026-11-03T15:36:31.659Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"bb3e9329-81a9-4baf-8625-b3aefb7f7212","cluster":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:36:31.846Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"33053bf0-5d01-4dea-987d-a6183115a401"}
{"level":"info","ts":"2026-11-03T15:36:31.846Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"33053bf0-5d01-4dea-987d-a6183115a401"}
{"level":"info","ts":"2026-11-03T15:36:31.846Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"33053bf0-5d01-4dea-987d-a6183115a401","1 head service found":"counter-auth-qmw8r-head-svc"}
{"level":"info","ts":"2026-11-03T15:36:31.846Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"33053bf0-5d01-4dea-987d-a6183115a401"}
{"level":"info","ts":"2026-11-03T15:36:31.896Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"33053bf0-5d01-4dea-987d-a6183115a401","error":"certificate ray-head-cert-counter-auth-qmw8r is not ready at current generation 1"}
{"level":"error","ts":"2026-11-03T15:36:31.896Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"33053bf0-5d01-4dea-987d-a6183115a401","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-counter-auth-qmw8r is not ready at current generation 1","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:36:31.896Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"33053bf0-5d01-4dea-987d-a6183115a401","error":"mTLS secrets not ready: certificate ray-head-cert-counter-auth-qmw8r is not ready at current generation 1"}
{"level":"info","ts":"2026-11-03T15:36:31.896Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"33053bf0-5d01-4dea-987d-a6183115a401","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"counter-auth-qmw8r","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:36:31.896Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"33053bf0-5d01-4dea-987d-a6183115a401","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"counter-auth-qmw8r","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:36:31.896Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"33053bf0-5d01-4dea-987d-a6183115a401","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"counter-auth-qmw8r","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:36:31.896Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"33053bf0-5d01-4dea-987d-a6183115a401"}
{"level":"error","ts":"2026-11-03T15:36:31.896Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"33053bf0-5d01-4dea-987d-a6183115a401","error":"mTLS secrets not ready: certificate ray-head-cert-counter-auth-qmw8r is not ready at current generation 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:36:32.111Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"bb3e9329-81a9-4baf-8625-b3aefb7f7212","certificate":"ray-head-cert-counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:36:32.111Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"bb3e9329-81a9-4baf-8625-b3aefb7f7212"}
{"level":"info","ts":"2026-11-03T15:36:32.537Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"e3261fbd-f346-4c0d-a561-9f3ddbe13975"}
{"level":"info","ts":"2026-11-03T15:36:32.537Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"e3261fbd-f346-4c0d-a561-9f3ddbe13975"}
{"level":"info","ts":"2026-11-03T15:36:32.537Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"e3261fbd-f346-4c0d-a561-9f3ddbe13975","1 head service found":"counter-auth-qmw8r-head-svc"}
{"level":"info","ts":"2026-11-03T15:36:32.537Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"e3261fbd-f346-4c0d-a561-9f3ddbe13975"}
{"level":"info","ts":"2026-11-03T15:36:32.587Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"e3261fbd-f346-4c0d-a561-9f3ddbe13975","error":"certificate ray-head-cert-counter-auth-qmw8r is not ready at current generation 1"}
{"level":"error","ts":"2026-11-03T15:36:32.587Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"e3261fbd-f346-4c0d-a561-9f3ddbe13975","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-counter-auth-qmw8r is not ready at current generation 1","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:36:32.587Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"e3261fbd-f346-4c0d-a561-9f3ddbe13975","error":"mTLS secrets not ready: certificate ray-head-cert-counter-auth-qmw8r is not ready at current generation 1"}
{"level":"info","ts":"2026-11-03T15:36:32.587Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"e3261fbd-f346-4c0d-a561-9f3ddbe13975","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"counter-auth-qmw8r","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:36:32.587Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"e3261fbd-f346-4c0d-a561-9f3ddbe13975","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"counter-auth-qmw8r","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:36:32.587Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"e3261fbd-f346-4c0d-a561-9f3ddbe13975","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"counter-auth-qmw8r","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:36:32.587Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"e3261fbd-f346-4c0d-a561-9f3ddbe13975"}
{"level":"error","ts":"2026-11-03T15:36:32.587Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"e3261fbd-f346-4c0d-a561-9f3ddbe13975","error":"mTLS secrets not ready: certificate ray-head-cert-counter-auth-qmw8r is not ready at current generation 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:36:33.868Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"14aa3d50-68bc-49a1-a30c-c0b148ae983f"}
{"level":"info","ts":"2026-11-03T15:36:33.868Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"14aa3d50-68bc-49a1-a30c-c0b148ae983f"}
{"level":"info","ts":"2026-11-03T15:36:33.868Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"14aa3d50-68bc-49a1-a30c-c0b148ae983f","1 head service found":"counter-auth-qmw8r-head-svc"}
{"level":"info","ts":"2026-11-03T15:36:33.868Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"14aa3d50-68bc-49a1-a30c-c0b148ae983f"}
{"level":"info","ts":"2026-11-03T15:36:33.969Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"14aa3d50-68bc-49a1-a30c-c0b148ae983f","mode":"IntegratedOAuth","cluster":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:36:33.969Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"14aa3d50-68bc-49a1-a30c-c0b148ae983f","cluster":"counter-auth-qmw8r","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-11-03T15:36:33.969Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"14aa3d50-68bc-49a1-a30c-c0b148ae983f","cluster":"counter-auth-qmw8r","authType":"IntegratedOAuth","serviceAccount":"counter-auth-qmw8r-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-11-03T15:36:33.969Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"14aa3d50-68bc-49a1-a30c-c0b148ae983f","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"counter-auth-qmw8r","ray.io/group":"headgroup","ray.io/identifier":"counter-auth-qmw8r-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-11-03T15:36:33.969Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"14aa3d50-68bc-49a1-a30c-c0b148ae983f","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-11-03T15:36:33.969Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"14aa3d50-68bc-49a1-a30c-c0b148ae983f","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-11-03T15:36:33.969Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"14aa3d50-68bc-49a1-a30c-c0b148ae983f","rayNodeType":"head","generatedCmd":"ulimit -n ${RAY_START_ULIMIT_OPEN_FILES:-65536}; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-11-03T15:36:33.969Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"14aa3d50-68bc-49a1-a30c-c0b148ae983f","enabled":true}
{"level":"info","ts":"2026-11-03T15:36:34.031Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"14aa3d50-68bc-49a1-a30c-c0b148ae983f","name":"counter-auth-qmw8r-head-b964f"}
{"level":"info","ts":"2026-11-03T15:36:34.031Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"14aa3d50-68bc-49a1-a30c-c0b148ae983f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:36:34.031Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"14aa3d50-68bc-49a1-a30c-c0b148ae983f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:36:34.031Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"14aa3d50-68bc-49a1-a30c-c0b148ae983f","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-11-03T15:36:34.031Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"14aa3d50-68bc-49a1-a30c-c0b148ae983f","group":"small-group","indices":{}}
{"level":"info","ts":"2026-11-03T15:36:34.031Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"14aa3d50-68bc-49a1-a30c-c0b148ae983f","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-11-03T15:36:34.132Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"14aa3d50-68bc-49a1-a30c-c0b148ae983f","creating worker for group":"small-group","index":0,"total":1,"replicaIndex":0}
{"level":"info","ts":"2026-11-03T15:36:34.132Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"14aa3d50-68bc-49a1-a30c-c0b148ae983f","nodeType":"worker","rayStartParams":{"address":"counter-auth-qmw8r-head-svc.test-ns-kbx4w.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-11-03T15:36:34.132Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"14aa3d50-68bc-49a1-a30c-c0b148ae983f","rayStartCmd":"ray start  --address=counter-auth-qmw8r-head-svc.test-ns-kbx4w.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-11-03T15:36:34.132Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"14aa3d50-68bc-49a1-a30c-c0b148ae983f","rayNodeType":"worker","generatedCmd":"ulimit -n ${RAY_START_ULIMIT_OPEN_FILES:-65536}; ray start  --address=counter-auth-qmw8r-head-svc.test-ns-kbx4w.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-11-03T15:36:34.132Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"14aa3d50-68bc-49a1-a30c-c0b148ae983f","enabled":true}
{"level":"info","ts":"2026-11-03T15:36:34.194Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"14aa3d50-68bc-49a1-a30c-c0b148ae983f","name":"counter-auth-qmw8r-small-group-worker-lqsm8"}
{"level":"info","ts":"2026-11-03T15:36:34.194Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"14aa3d50-68bc-49a1-a30c-c0b148ae983f","podName":"counter-auth-qmw8r-head-b964f","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:36:34.194Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"14aa3d50-68bc-49a1-a30c-c0b148ae983f","name":"counter-auth-qmw8r","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:34Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:36:34.256Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"6c676794-af81-45d4-a8db-8f628f0fb75d","cluster":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:36:34.256Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"ea8ff633-49ac-4b17-9e21-aed5037915c6"}
{"level":"info","ts":"2026-11-03T15:36:34.256Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"ea8ff633-49ac-4b17-9e21-aed5037915c6"}
{"level":"info","ts":"2026-11-03T15:36:34.256Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"ea8ff633-49ac-4b17-9e21-aed5037915c6","1 head service found":"counter-auth-qmw8r-head-svc"}
{"level":"info","ts":"2026-11-03T15:36:34.256Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"ea8ff633-49ac-4b17-9e21-aed5037915c6","Found 1 head Pod":"counter-auth-qmw8r-head-b964f","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:36:34.256Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"ea8ff633-49ac-4b17-9e21-aed5037915c6","head Pod":"counter-auth-qmw8r-head-b964f","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-qmw8r-head-b964f. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:36:34.256Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"ea8ff633-49ac-4b17-9e21-aed5037915c6","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:36:34.256Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"975d2aca-5f71-4605-91a2-3b6dff27636c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:36:34.256Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"975d2aca-5f71-4605-91a2-3b6dff27636c","name":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:36:34.257Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"975d2aca-5f71-4605-91a2-3b6dff27636c","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:36:34.257Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"975d2aca-5f71-4605-91a2-3b6dff27636c"}
{"level":"info","ts":"2026-11-03T15:36:34.257Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"975d2aca-5f71-4605-91a2-3b6dff27636c","RayCluster":"counter-auth-qmw8r","State":""}
{"level":"info","ts":"2026-11-03T15:36:34.256Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"ea8ff633-49ac-4b17-9e21-aed5037915c6","worker Pod":"counter-auth-qmw8r-small-group-worker-lqsm8","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-qmw8r-small-group-worker-lqsm8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:36:34.257Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"ea8ff633-49ac-4b17-9e21-aed5037915c6","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:36:34.257Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"ea8ff633-49ac-4b17-9e21-aed5037915c6","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:36:34.257Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"ea8ff633-49ac-4b17-9e21-aed5037915c6","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:36:34.257Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"ea8ff633-49ac-4b17-9e21-aed5037915c6","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:36:34.257Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"975d2aca-5f71-4605-91a2-3b6dff27636c","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:30Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:34Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:36:34.257Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"975d2aca-5f71-4605-91a2-3b6dff27636c","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:36:34.257Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"ea8ff633-49ac-4b17-9e21-aed5037915c6","podName":"counter-auth-qmw8r-head-b964f","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:36:34.257Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"ea8ff633-49ac-4b17-9e21-aed5037915c6","podName":"counter-auth-qmw8r-head-b964f","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:36:34.257Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"ea8ff633-49ac-4b17-9e21-aed5037915c6","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:36:34.257Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"ea8ff633-49ac-4b17-9e21-aed5037915c6","seconds":300}
{"level":"info","ts":"2026-11-03T15:36:34.266Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"3a5f849e-dae3-43a1-bd96-1ca5154b085d"}
{"level":"info","ts":"2026-11-03T15:36:34.266Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"3a5f849e-dae3-43a1-bd96-1ca5154b085d"}
{"level":"info","ts":"2026-11-03T15:36:34.266Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"3a5f849e-dae3-43a1-bd96-1ca5154b085d","1 head service found":"counter-auth-qmw8r-head-svc"}
{"level":"info","ts":"2026-11-03T15:36:34.266Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"3a5f849e-dae3-43a1-bd96-1ca5154b085d","Found 1 head Pod":"counter-auth-qmw8r-head-b964f","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:36:34.266Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"3a5f849e-dae3-43a1-bd96-1ca5154b085d","head Pod":"counter-auth-qmw8r-head-b964f","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-qmw8r-head-b964f. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:36:34.266Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"3a5f849e-dae3-43a1-bd96-1ca5154b085d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:36:34.266Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"3a5f849e-dae3-43a1-bd96-1ca5154b085d","worker Pod":"counter-auth-qmw8r-small-group-worker-lqsm8","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-qmw8r-small-group-worker-lqsm8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:36:34.266Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"3a5f849e-dae3-43a1-bd96-1ca5154b085d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:36:34.266Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"3a5f849e-dae3-43a1-bd96-1ca5154b085d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:36:34.266Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"3a5f849e-dae3-43a1-bd96-1ca5154b085d","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:36:34.266Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"3a5f849e-dae3-43a1-bd96-1ca5154b085d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:36:34.266Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"3a5f849e-dae3-43a1-bd96-1ca5154b085d","podName":"counter-auth-qmw8r-head-b964f","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:36:34.266Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"3a5f849e-dae3-43a1-bd96-1ca5154b085d","podName":"counter-auth-qmw8r-head-b964f","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:36:34.266Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"3a5f849e-dae3-43a1-bd96-1ca5154b085d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:36:34.266Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"3a5f849e-dae3-43a1-bd96-1ca5154b085d","seconds":300}
{"level":"info","ts":"2026-11-03T15:36:34.318Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"d3aee626-5b6b-47b2-b333-91520fd7b108","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:36:34.318Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"d3aee626-5b6b-47b2-b333-91520fd7b108","name":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:36:34.318Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"d3aee626-5b6b-47b2-b333-91520fd7b108","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:36:34.318Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"d3aee626-5b6b-47b2-b333-91520fd7b108"}
{"level":"info","ts":"2026-11-03T15:36:34.318Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"d3aee626-5b6b-47b2-b333-91520fd7b108","RayCluster":"counter-auth-qmw8r","State":""}
{"level":"info","ts":"2026-11-03T15:36:34.318Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"d3aee626-5b6b-47b2-b333-91520fd7b108","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:34Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:34Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:36:34.328Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"ffcc1ae0-5273-4af8-b1e4-c954b6bd7908"}
{"level":"info","ts":"2026-11-03T15:36:34.328Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"ffcc1ae0-5273-4af8-b1e4-c954b6bd7908"}
{"level":"info","ts":"2026-11-03T15:36:34.328Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"ffcc1ae0-5273-4af8-b1e4-c954b6bd7908","1 head service found":"counter-auth-qmw8r-head-svc"}
{"level":"info","ts":"2026-11-03T15:36:34.329Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"ffcc1ae0-5273-4af8-b1e4-c954b6bd7908","Found 1 head Pod":"counter-auth-qmw8r-head-b964f","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:36:34.329Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"ffcc1ae0-5273-4af8-b1e4-c954b6bd7908","head Pod":"counter-auth-qmw8r-head-b964f","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-qmw8r-head-b964f. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:36:34.329Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"ffcc1ae0-5273-4af8-b1e4-c954b6bd7908","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:36:34.329Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"ffcc1ae0-5273-4af8-b1e4-c954b6bd7908","worker Pod":"counter-auth-qmw8r-small-group-worker-lqsm8","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-qmw8r-small-group-worker-lqsm8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:36:34.329Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"ffcc1ae0-5273-4af8-b1e4-c954b6bd7908","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:36:34.329Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"ffcc1ae0-5273-4af8-b1e4-c954b6bd7908","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:36:34.329Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"ffcc1ae0-5273-4af8-b1e4-c954b6bd7908","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:36:34.329Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"ffcc1ae0-5273-4af8-b1e4-c954b6bd7908","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:36:34.329Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"ffcc1ae0-5273-4af8-b1e4-c954b6bd7908","podName":"counter-auth-qmw8r-head-b964f","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:36:34.329Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"ffcc1ae0-5273-4af8-b1e4-c954b6bd7908","podName":"counter-auth-qmw8r-head-b964f","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:36:34.329Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"ffcc1ae0-5273-4af8-b1e4-c954b6bd7908","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:36:34.329Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"ffcc1ae0-5273-4af8-b1e4-c954b6bd7908","seconds":300}
{"level":"info","ts":"2026-11-03T15:36:34.790Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"7ca15388-a4b8-4955-ade6-84edb1adb5b6"}
{"level":"info","ts":"2026-11-03T15:36:34.790Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"7ca15388-a4b8-4955-ade6-84edb1adb5b6"}
{"level":"info","ts":"2026-11-03T15:36:34.790Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"7ca15388-a4b8-4955-ade6-84edb1adb5b6","1 head service found":"counter-auth-qmw8r-head-svc"}
{"level":"info","ts":"2026-11-03T15:36:34.790Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"7ca15388-a4b8-4955-ade6-84edb1adb5b6","Found 1 head Pod":"counter-auth-qmw8r-head-b964f","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:36:34.790Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"7ca15388-a4b8-4955-ade6-84edb1adb5b6","head Pod":"counter-auth-qmw8r-head-b964f","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-qmw8r-head-b964f. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:36:34.790Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"7ca15388-a4b8-4955-ade6-84edb1adb5b6","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:36:34.790Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"7ca15388-a4b8-4955-ade6-84edb1adb5b6","worker Pod":"counter-auth-qmw8r-small-group-worker-lqsm8","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-qmw8r-small-group-worker-lqsm8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:36:34.790Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"7ca15388-a4b8-4955-ade6-84edb1adb5b6","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:36:34.790Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"7ca15388-a4b8-4955-ade6-84edb1adb5b6","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:36:34.790Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"7ca15388-a4b8-4955-ade6-84edb1adb5b6","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:36:34.790Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"7ca15388-a4b8-4955-ade6-84edb1adb5b6","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:36:34.790Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"7ca15388-a4b8-4955-ade6-84edb1adb5b6","podName":"counter-auth-qmw8r-head-b964f","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:36:34.790Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"7ca15388-a4b8-4955-ade6-84edb1adb5b6","podName":"counter-auth-qmw8r-head-b964f","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:36:34.791Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"7ca15388-a4b8-4955-ade6-84edb1adb5b6","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:36:34.791Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"7ca15388-a4b8-4955-ade6-84edb1adb5b6","seconds":300}
{"level":"info","ts":"2026-11-03T15:36:35.304Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"1921c5d9-6637-40bc-b975-ba1438b80b65"}
{"level":"info","ts":"2026-11-03T15:36:35.304Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"1921c5d9-6637-40bc-b975-ba1438b80b65"}
{"level":"info","ts":"2026-11-03T15:36:35.304Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"1921c5d9-6637-40bc-b975-ba1438b80b65","1 head service found":"counter-auth-qmw8r-head-svc"}
{"level":"info","ts":"2026-11-03T15:36:35.304Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"1921c5d9-6637-40bc-b975-ba1438b80b65","Found 1 head Pod":"counter-auth-qmw8r-head-b964f","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:36:35.304Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"1921c5d9-6637-40bc-b975-ba1438b80b65","head Pod":"counter-auth-qmw8r-head-b964f","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-qmw8r-head-b964f. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:36:35.304Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"1921c5d9-6637-40bc-b975-ba1438b80b65","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:36:35.304Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"1921c5d9-6637-40bc-b975-ba1438b80b65","worker Pod":"counter-auth-qmw8r-small-group-worker-lqsm8","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-qmw8r-small-group-worker-lqsm8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:36:35.304Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"1921c5d9-6637-40bc-b975-ba1438b80b65","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:36:35.304Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"1921c5d9-6637-40bc-b975-ba1438b80b65","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:36:35.304Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"1921c5d9-6637-40bc-b975-ba1438b80b65","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:36:35.304Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"1921c5d9-6637-40bc-b975-ba1438b80b65","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:36:35.305Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"1921c5d9-6637-40bc-b975-ba1438b80b65","podName":"counter-auth-qmw8r-head-b964f","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:36:35.305Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"1921c5d9-6637-40bc-b975-ba1438b80b65","podName":"counter-auth-qmw8r-head-b964f","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:36:35.305Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"1921c5d9-6637-40bc-b975-ba1438b80b65","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:36:35.305Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"1921c5d9-6637-40bc-b975-ba1438b80b65","seconds":300}
{"level":"info","ts":"2026-11-03T15:36:35.445Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"7beb0e56-c57e-4214-82db-4a25594bc92e"}
{"level":"info","ts":"2026-11-03T15:36:35.445Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"7beb0e56-c57e-4214-82db-4a25594bc92e"}
{"level":"info","ts":"2026-11-03T15:36:35.445Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"7beb0e56-c57e-4214-82db-4a25594bc92e","1 head service found":"counter-auth-qmw8r-head-svc"}
{"level":"info","ts":"2026-11-03T15:36:35.445Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"7beb0e56-c57e-4214-82db-4a25594bc92e","Found 1 head Pod":"counter-auth-qmw8r-head-b964f","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:36:35.445Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"7beb0e56-c57e-4214-82db-4a25594bc92e","head Pod":"counter-auth-qmw8r-head-b964f","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-qmw8r-head-b964f. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:36:35.445Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"7beb0e56-c57e-4214-82db-4a25594bc92e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:36:35.445Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"7beb0e56-c57e-4214-82db-4a25594bc92e","worker Pod":"counter-auth-qmw8r-small-group-worker-lqsm8","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-qmw8r-small-group-worker-lqsm8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:36:35.445Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"7beb0e56-c57e-4214-82db-4a25594bc92e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:36:35.445Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"7beb0e56-c57e-4214-82db-4a25594bc92e","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:36:35.445Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"7beb0e56-c57e-4214-82db-4a25594bc92e","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:36:35.445Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"7beb0e56-c57e-4214-82db-4a25594bc92e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:36:35.445Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"7beb0e56-c57e-4214-82db-4a25594bc92e","podName":"counter-auth-qmw8r-head-b964f","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:36:35.445Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"7beb0e56-c57e-4214-82db-4a25594bc92e","podName":"counter-auth-qmw8r-head-b964f","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:36:35.445Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"7beb0e56-c57e-4214-82db-4a25594bc92e","name":"counter-auth-qmw8r","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:34Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:36:35.506Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"df444d63-4ab0-4362-b608-a5ef58f3aebe","cluster":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:36:35.507Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"37897510-31f7-44cf-bce0-87e94a195318","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:36:35.507Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"37897510-31f7-44cf-bce0-87e94a195318","name":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:36:35.507Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"37897510-31f7-44cf-bce0-87e94a195318","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:36:35.507Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"37897510-31f7-44cf-bce0-87e94a195318"}
{"level":"info","ts":"2026-11-03T15:36:35.507Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"37897510-31f7-44cf-bce0-87e94a195318","RayCluster":"counter-auth-qmw8r","State":""}
{"level":"info","ts":"2026-11-03T15:36:35.507Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"37897510-31f7-44cf-bce0-87e94a195318","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:34Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:36:35.507Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"37897510-31f7-44cf-bce0-87e94a195318","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:36:35.568Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"d3df4e22-7ac1-4752-8b41-34577c2943c9","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:36:35.568Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"d3df4e22-7ac1-4752-8b41-34577c2943c9","name":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:36:35.568Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"d3df4e22-7ac1-4752-8b41-34577c2943c9","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:36:35.568Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"d3df4e22-7ac1-4752-8b41-34577c2943c9"}
{"level":"info","ts":"2026-11-03T15:36:35.568Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"d3df4e22-7ac1-4752-8b41-34577c2943c9","RayCluster":"counter-auth-qmw8r","State":""}
{"level":"info","ts":"2026-11-03T15:36:35.568Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"d3df4e22-7ac1-4752-8b41-34577c2943c9","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:36:35.809Z","logger":"controllers.RayClusterMTLS","msg":"Updating head certificate","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"df444d63-4ab0-4362-b608-a5ef58f3aebe","certificate":"ray-head-cert-counter-auth-qmw8r","ipAddresses":["10.134.0.53","127.0.0.1"]}
{"level":"info","ts":"2026-11-03T15:36:36.230Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"df444d63-4ab0-4362-b608-a5ef58f3aebe","certificate":"ray-head-cert-counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:36:36.230Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"df444d63-4ab0-4362-b608-a5ef58f3aebe"}
{"level":"info","ts":"2026-11-03T15:36:36.246Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"8df9e950-3cdb-490b-b95c-1472c4e2a081"}
{"level":"info","ts":"2026-11-03T15:36:36.246Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"8df9e950-3cdb-490b-b95c-1472c4e2a081"}
{"level":"info","ts":"2026-11-03T15:36:36.246Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"8df9e950-3cdb-490b-b95c-1472c4e2a081","1 head service found":"counter-auth-qmw8r-head-svc"}
{"level":"info","ts":"2026-11-03T15:36:36.247Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"8df9e950-3cdb-490b-b95c-1472c4e2a081","Found 1 head Pod":"counter-auth-qmw8r-head-b964f","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:36:36.247Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"8df9e950-3cdb-490b-b95c-1472c4e2a081","head Pod":"counter-auth-qmw8r-head-b964f","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-qmw8r-head-b964f. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:36:36.247Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"8df9e950-3cdb-490b-b95c-1472c4e2a081","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:36:36.247Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"8df9e950-3cdb-490b-b95c-1472c4e2a081","worker Pod":"counter-auth-qmw8r-small-group-worker-lqsm8","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-qmw8r-small-group-worker-lqsm8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:36:36.247Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"8df9e950-3cdb-490b-b95c-1472c4e2a081","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:36:36.247Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"8df9e950-3cdb-490b-b95c-1472c4e2a081","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:36:36.247Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"8df9e950-3cdb-490b-b95c-1472c4e2a081","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:36:36.247Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"8df9e950-3cdb-490b-b95c-1472c4e2a081","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:36:36.247Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"8df9e950-3cdb-490b-b95c-1472c4e2a081","podName":"counter-auth-qmw8r-head-b964f","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:36:36.247Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"8df9e950-3cdb-490b-b95c-1472c4e2a081","podName":"counter-auth-qmw8r-head-b964f","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:36:36.247Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"8df9e950-3cdb-490b-b95c-1472c4e2a081","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:36:36.247Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"8df9e950-3cdb-490b-b95c-1472c4e2a081","seconds":300}
{"level":"info","ts":"2026-11-03T15:36:38.568Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"ec84d361-07db-49f4-bfcb-c1b6e19b9ce8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:36:38.568Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"ec84d361-07db-49f4-bfcb-c1b6e19b9ce8","name":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:36:38.568Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"ec84d361-07db-49f4-bfcb-c1b6e19b9ce8","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:36:38.569Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"ec84d361-07db-49f4-bfcb-c1b6e19b9ce8"}
{"level":"info","ts":"2026-11-03T15:36:38.569Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"ec84d361-07db-49f4-bfcb-c1b6e19b9ce8","RayCluster":"counter-auth-qmw8r","State":""}
{"level":"info","ts":"2026-11-03T15:36:38.569Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"ec84d361-07db-49f4-bfcb-c1b6e19b9ce8","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:36:41.570Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"2b0fb85f-e072-4db5-a68d-00125ba4e9e5","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:36:41.570Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"2b0fb85f-e072-4db5-a68d-00125ba4e9e5","name":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:36:41.570Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"2b0fb85f-e072-4db5-a68d-00125ba4e9e5","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:36:41.570Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"2b0fb85f-e072-4db5-a68d-00125ba4e9e5"}
{"level":"info","ts":"2026-11-03T15:36:41.570Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"2b0fb85f-e072-4db5-a68d-00125ba4e9e5","RayCluster":"counter-auth-qmw8r","State":""}
{"level":"info","ts":"2026-11-03T15:36:41.570Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"2b0fb85f-e072-4db5-a68d-00125ba4e9e5","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:36:43.938Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"f31ef5ad-7bc4-4a30-9c58-98cf1bdcc3c1","cluster":"long-running-hvml9"}
{"level":"info","ts":"2026-11-03T15:36:44.570Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"b3287345-4235-46e7-8ae8-64ef2060e28f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:36:44.571Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"b3287345-4235-46e7-8ae8-64ef2060e28f","name":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:36:44.571Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"b3287345-4235-46e7-8ae8-64ef2060e28f","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:36:44.571Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"b3287345-4235-46e7-8ae8-64ef2060e28f"}
{"level":"info","ts":"2026-11-03T15:36:44.571Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"b3287345-4235-46e7-8ae8-64ef2060e28f","RayCluster":"counter-auth-qmw8r","State":""}
{"level":"info","ts":"2026-11-03T15:36:44.571Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"b3287345-4235-46e7-8ae8-64ef2060e28f","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:36:47.571Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"7fa0813c-fd59-439c-a042-5b0004600c5a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:36:47.571Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"7fa0813c-fd59-439c-a042-5b0004600c5a","name":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:36:47.571Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"7fa0813c-fd59-439c-a042-5b0004600c5a","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:36:47.572Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"7fa0813c-fd59-439c-a042-5b0004600c5a"}
{"level":"info","ts":"2026-11-03T15:36:47.572Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"7fa0813c-fd59-439c-a042-5b0004600c5a","RayCluster":"counter-auth-qmw8r","State":""}
{"level":"info","ts":"2026-11-03T15:36:47.572Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"7fa0813c-fd59-439c-a042-5b0004600c5a","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:36:50.572Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"7f496758-6007-49e5-9d95-b10bbd29475c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:36:50.572Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"7f496758-6007-49e5-9d95-b10bbd29475c","name":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:36:50.572Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"7f496758-6007-49e5-9d95-b10bbd29475c","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:36:50.572Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"7f496758-6007-49e5-9d95-b10bbd29475c"}
{"level":"info","ts":"2026-11-03T15:36:50.572Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"7f496758-6007-49e5-9d95-b10bbd29475c","RayCluster":"counter-auth-qmw8r","State":""}
{"level":"info","ts":"2026-11-03T15:36:50.572Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"7f496758-6007-49e5-9d95-b10bbd29475c","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:36:53.573Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"86e6b173-94e2-415c-90cf-83e650864509","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:36:53.573Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"86e6b173-94e2-415c-90cf-83e650864509","name":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:36:53.573Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"86e6b173-94e2-415c-90cf-83e650864509","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:36:53.573Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"86e6b173-94e2-415c-90cf-83e650864509"}
{"level":"info","ts":"2026-11-03T15:36:53.573Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"86e6b173-94e2-415c-90cf-83e650864509","RayCluster":"counter-auth-qmw8r","State":""}
{"level":"info","ts":"2026-11-03T15:36:53.573Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"86e6b173-94e2-415c-90cf-83e650864509","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:36:56.574Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"c263cce5-4c9e-41f0-9257-49b5d19b6762","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:36:56.574Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"c263cce5-4c9e-41f0-9257-49b5d19b6762","name":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:36:56.574Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"c263cce5-4c9e-41f0-9257-49b5d19b6762","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:36:56.575Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"c263cce5-4c9e-41f0-9257-49b5d19b6762"}
{"level":"info","ts":"2026-11-03T15:36:56.575Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"c263cce5-4c9e-41f0-9257-49b5d19b6762","RayCluster":"counter-auth-qmw8r","State":""}
{"level":"info","ts":"2026-11-03T15:36:56.575Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"c263cce5-4c9e-41f0-9257-49b5d19b6762","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:36:59.575Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"a5a301a8-6a83-438b-b719-b7e9e2db3711","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:36:59.575Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"a5a301a8-6a83-438b-b719-b7e9e2db3711","name":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:36:59.575Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"a5a301a8-6a83-438b-b719-b7e9e2db3711","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:36:59.576Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"a5a301a8-6a83-438b-b719-b7e9e2db3711"}
{"level":"info","ts":"2026-11-03T15:36:59.576Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"a5a301a8-6a83-438b-b719-b7e9e2db3711","RayCluster":"counter-auth-qmw8r","State":""}
{"level":"info","ts":"2026-11-03T15:36:59.576Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"a5a301a8-6a83-438b-b719-b7e9e2db3711","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:37:02.576Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"52e1b74b-4a19-4529-97f5-266e2d4a3fb7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:37:02.576Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"52e1b74b-4a19-4529-97f5-266e2d4a3fb7","name":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:37:02.576Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"52e1b74b-4a19-4529-97f5-266e2d4a3fb7","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:37:02.576Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"52e1b74b-4a19-4529-97f5-266e2d4a3fb7"}
{"level":"info","ts":"2026-11-03T15:37:02.576Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"52e1b74b-4a19-4529-97f5-266e2d4a3fb7","RayCluster":"counter-auth-qmw8r","State":""}
{"level":"info","ts":"2026-11-03T15:37:02.576Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"52e1b74b-4a19-4529-97f5-266e2d4a3fb7","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:37:05.577Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"49732f35-3387-4ad5-b95f-2f488ee65451","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:37:05.577Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"49732f35-3387-4ad5-b95f-2f488ee65451","name":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:37:05.577Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"49732f35-3387-4ad5-b95f-2f488ee65451","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:37:05.577Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"49732f35-3387-4ad5-b95f-2f488ee65451"}
{"level":"info","ts":"2026-11-03T15:37:05.577Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"49732f35-3387-4ad5-b95f-2f488ee65451","RayCluster":"counter-auth-qmw8r","State":""}
{"level":"info","ts":"2026-11-03T15:37:05.577Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"49732f35-3387-4ad5-b95f-2f488ee65451","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:37:06.231Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"06c9fff1-77b9-47c8-8218-29d545bf1170","cluster":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:37:06.686Z","logger":"controllers.RayClusterMTLS","msg":"Updating worker certificate","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"06c9fff1-77b9-47c8-8218-29d545bf1170","certificate":"ray-worker-cert-counter-auth-qmw8r","ipAddresses":["10.132.0.48","127.0.0.1"]}
{"level":"info","ts":"2026-11-03T15:37:07.012Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"06c9fff1-77b9-47c8-8218-29d545bf1170","certificate":"ray-worker-cert-counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:37:07.013Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"06c9fff1-77b9-47c8-8218-29d545bf1170"}
{"level":"info","ts":"2026-11-03T15:37:08.579Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"b57ff430-4078-4533-b09b-be335a38b226","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:37:08.579Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"b57ff430-4078-4533-b09b-be335a38b226","name":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:37:08.579Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"b57ff430-4078-4533-b09b-be335a38b226","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:37:08.579Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"b57ff430-4078-4533-b09b-be335a38b226"}
{"level":"info","ts":"2026-11-03T15:37:08.579Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"b57ff430-4078-4533-b09b-be335a38b226","RayCluster":"counter-auth-qmw8r","State":""}
{"level":"info","ts":"2026-11-03T15:37:08.579Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"b57ff430-4078-4533-b09b-be335a38b226","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:37:11.580Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"bc1f3b11-f053-4849-98a8-9dbf55ed0032","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:37:11.580Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"bc1f3b11-f053-4849-98a8-9dbf55ed0032","name":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:37:11.580Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"bc1f3b11-f053-4849-98a8-9dbf55ed0032","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:37:11.580Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"bc1f3b11-f053-4849-98a8-9dbf55ed0032"}
{"level":"info","ts":"2026-11-03T15:37:11.580Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"bc1f3b11-f053-4849-98a8-9dbf55ed0032","RayCluster":"counter-auth-qmw8r","State":""}
{"level":"info","ts":"2026-11-03T15:37:11.580Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"bc1f3b11-f053-4849-98a8-9dbf55ed0032","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:37:14.581Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"4fcaf202-84df-4ffe-aeb6-b909eab4b593","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:37:14.581Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"4fcaf202-84df-4ffe-aeb6-b909eab4b593","name":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:37:14.581Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"4fcaf202-84df-4ffe-aeb6-b909eab4b593","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:37:14.581Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"4fcaf202-84df-4ffe-aeb6-b909eab4b593"}
{"level":"info","ts":"2026-11-03T15:37:14.581Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"4fcaf202-84df-4ffe-aeb6-b909eab4b593","RayCluster":"counter-auth-qmw8r","State":""}
{"level":"info","ts":"2026-11-03T15:37:14.581Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"4fcaf202-84df-4ffe-aeb6-b909eab4b593","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:37:17.582Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"664dba5d-eee0-483f-93f9-d9444f0f9791","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:37:17.582Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"664dba5d-eee0-483f-93f9-d9444f0f9791","name":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:37:17.582Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"664dba5d-eee0-483f-93f9-d9444f0f9791","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:37:17.582Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"664dba5d-eee0-483f-93f9-d9444f0f9791"}
{"level":"info","ts":"2026-11-03T15:37:17.582Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"664dba5d-eee0-483f-93f9-d9444f0f9791","RayCluster":"counter-auth-qmw8r","State":""}
{"level":"info","ts":"2026-11-03T15:37:17.582Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"664dba5d-eee0-483f-93f9-d9444f0f9791","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:37:20.583Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"ea1c15bb-3794-4cc7-99a8-d87755139936","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:37:20.583Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"ea1c15bb-3794-4cc7-99a8-d87755139936","name":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:37:20.583Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"ea1c15bb-3794-4cc7-99a8-d87755139936","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:37:20.583Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"ea1c15bb-3794-4cc7-99a8-d87755139936"}
{"level":"info","ts":"2026-11-03T15:37:20.583Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"ea1c15bb-3794-4cc7-99a8-d87755139936","RayCluster":"counter-auth-qmw8r","State":""}
{"level":"info","ts":"2026-11-03T15:37:20.583Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"ea1c15bb-3794-4cc7-99a8-d87755139936","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:37:23.584Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"be81dd24-2f3e-46bf-a72e-83b6e6a447cb","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:37:23.584Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"be81dd24-2f3e-46bf-a72e-83b6e6a447cb","name":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:37:23.584Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"be81dd24-2f3e-46bf-a72e-83b6e6a447cb","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:37:23.585Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"be81dd24-2f3e-46bf-a72e-83b6e6a447cb"}
{"level":"info","ts":"2026-11-03T15:37:23.585Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"be81dd24-2f3e-46bf-a72e-83b6e6a447cb","RayCluster":"counter-auth-qmw8r","State":""}
{"level":"info","ts":"2026-11-03T15:37:23.585Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"be81dd24-2f3e-46bf-a72e-83b6e6a447cb","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:37:26.586Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"34d8cf55-9fa2-4735-bbce-658c1d0a29ad","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:37:26.586Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"34d8cf55-9fa2-4735-bbce-658c1d0a29ad","name":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:37:26.586Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"34d8cf55-9fa2-4735-bbce-658c1d0a29ad","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:37:26.586Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"34d8cf55-9fa2-4735-bbce-658c1d0a29ad"}
{"level":"info","ts":"2026-11-03T15:37:26.586Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"34d8cf55-9fa2-4735-bbce-658c1d0a29ad","RayCluster":"counter-auth-qmw8r","State":""}
{"level":"info","ts":"2026-11-03T15:37:26.586Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"34d8cf55-9fa2-4735-bbce-658c1d0a29ad","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:37:29.587Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"6d6f9458-3745-4d63-b85a-fc62435f44d3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:37:29.587Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"6d6f9458-3745-4d63-b85a-fc62435f44d3","name":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:37:29.587Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"6d6f9458-3745-4d63-b85a-fc62435f44d3","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:37:29.587Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"6d6f9458-3745-4d63-b85a-fc62435f44d3"}
{"level":"info","ts":"2026-11-03T15:37:29.587Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"6d6f9458-3745-4d63-b85a-fc62435f44d3","RayCluster":"counter-auth-qmw8r","State":""}
{"level":"info","ts":"2026-11-03T15:37:29.587Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"6d6f9458-3745-4d63-b85a-fc62435f44d3","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:37:32.588Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"b28d64eb-45f7-4475-b9d0-14c6d947973e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:37:32.588Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"b28d64eb-45f7-4475-b9d0-14c6d947973e","name":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:37:32.588Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"b28d64eb-45f7-4475-b9d0-14c6d947973e","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:37:32.588Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"b28d64eb-45f7-4475-b9d0-14c6d947973e"}
{"level":"info","ts":"2026-11-03T15:37:32.588Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"b28d64eb-45f7-4475-b9d0-14c6d947973e","RayCluster":"counter-auth-qmw8r","State":""}
{"level":"info","ts":"2026-11-03T15:37:32.588Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"b28d64eb-45f7-4475-b9d0-14c6d947973e","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:37:35.589Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"5dafe1d2-ed2a-4907-9f6f-cd071f3c8cac","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:37:35.589Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"5dafe1d2-ed2a-4907-9f6f-cd071f3c8cac","name":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:37:35.589Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"5dafe1d2-ed2a-4907-9f6f-cd071f3c8cac","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:37:35.589Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"5dafe1d2-ed2a-4907-9f6f-cd071f3c8cac"}
{"level":"info","ts":"2026-11-03T15:37:35.589Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"5dafe1d2-ed2a-4907-9f6f-cd071f3c8cac","RayCluster":"counter-auth-qmw8r","State":""}
{"level":"info","ts":"2026-11-03T15:37:35.589Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"5dafe1d2-ed2a-4907-9f6f-cd071f3c8cac","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:37:37.014Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"c3abb9be-90d9-4080-a9df-91139089165f","cluster":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:37:38.590Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"7e19bf9d-e6d9-4da1-9fca-801683667292","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:37:38.590Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"7e19bf9d-e6d9-4da1-9fca-801683667292","name":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:37:38.590Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"7e19bf9d-e6d9-4da1-9fca-801683667292","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:37:38.590Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"7e19bf9d-e6d9-4da1-9fca-801683667292"}
{"level":"info","ts":"2026-11-03T15:37:38.590Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"7e19bf9d-e6d9-4da1-9fca-801683667292","RayCluster":"counter-auth-qmw8r","State":""}
{"level":"info","ts":"2026-11-03T15:37:38.590Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"7e19bf9d-e6d9-4da1-9fca-801683667292","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:37:40.001Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"5bb9f3af-d570-4218-ba4c-c742dbba25a4","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:37:40.001Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"5bb9f3af-d570-4218-ba4c-c742dbba25a4","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Failed","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":600,"Status.endTime":"2026-11-03 15:27:38 +0000 UTC","Now":"2026-11-03T15:37:40.001Z","ShutdownTime":"2026-11-03T15:37:38.000Z"}
{"level":"info","ts":"2026-11-03T15:37:40.106Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob is deleted","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"5bb9f3af-d570-4218-ba4c-c742dbba25a4","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:37:40.106Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"5bb9f3af-d570-4218-ba4c-c742dbba25a4","isClusterDeleted":false}
{"level":"info","ts":"2026-11-03T15:37:40.106Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"5bb9f3af-d570-4218-ba4c-c742dbba25a4","deletionMechanism":"ShutdownAfterJobFinishes","RayCluster":"long-running-hvml9"}
{"level":"info","ts":"2026-11-03T15:37:40.107Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"beb1ffa0-be6f-4d3e-b1ee-5580e1198223","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:37:40.107Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"beb1ffa0-be6f-4d3e-b1ee-5580e1198223","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Failed","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":600,"Status.endTime":"2026-11-03 15:27:38 +0000 UTC","Now":"2026-11-03T15:37:40.107Z","ShutdownTime":"2026-11-03T15:37:38.000Z"}
{"level":"info","ts":"2026-11-03T15:37:40.107Z","logger":"controllers.RayJob","msg":"The deletion of the associated RayCluster for RayJob is ongoing.","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"beb1ffa0-be6f-4d3e-b1ee-5580e1198223","RayCluster":"long-running-hvml9"}
{"level":"info","ts":"2026-11-03T15:37:40.107Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"beb1ffa0-be6f-4d3e-b1ee-5580e1198223","isClusterDeleted":false}
{"level":"info","ts":"2026-11-03T15:37:40.107Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"beb1ffa0-be6f-4d3e-b1ee-5580e1198223","deletionMechanism":"ShutdownAfterJobFinishes","RayCluster":"long-running-hvml9"}
{"level":"info","ts":"2026-11-03T15:37:40.107Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"reconcileID":"44e7b4cd-7080-4cf4-8c6f-40ec48b25ab3"}
{"level":"info","ts":"2026-11-03T15:37:40.107Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"long-running-hvml9","reconcileID":"4f4b8e52-77bd-4724-8ff5-55c6f14e640f","namespacedName":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:37:40.107Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"long-running-hvml9","reconcileID":"4f4b8e52-77bd-4724-8ff5-55c6f14e640f","cluster":"long-running-hvml9"}
{"level":"info","ts":"2026-11-03T15:37:40.107Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"long-running-hvml9","reconcileID":"4f4b8e52-77bd-4724-8ff5-55c6f14e640f","cluster":"long-running-hvml9"}
{"level":"info","ts":"2026-11-03T15:37:40.107Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"long-running-hvml9","reconcileID":"4f4b8e52-77bd-4724-8ff5-55c6f14e640f","namespace":"test-ns-kbx4w","otherAuthClusters":1,"action":"will retain"}
{"level":"info","ts":"2026-11-03T15:37:40.107Z","logger":"authentication-controller","msg":"ReferenceGrant retained (other clusters with auth exist)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"long-running-hvml9","reconcileID":"4f4b8e52-77bd-4724-8ff5-55c6f14e640f","name":"kuberay-gateway-access","namespace":"test-ns-kbx4w","remainingClusters":1}
{"level":"info","ts":"2026-11-03T15:37:40.160Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"long-running-hvml9","reconcileID":"4f4b8e52-77bd-4724-8ff5-55c6f14e640f","name":"test-ns-kbx4w-long-running-hvml9","namespace":"opendatahub"}
{"level":"info","ts":"2026-11-03T15:37:40.212Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"long-running-hvml9","reconcileID":"4f4b8e52-77bd-4724-8ff5-55c6f14e640f","configMap":"kube-rbac-proxy-config-long-running-hvml9"}
{"level":"info","ts":"2026-11-03T15:37:40.264Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"long-running-hvml9","reconcileID":"4f4b8e52-77bd-4724-8ff5-55c6f14e640f","serviceAccount":"long-running-hvml9-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-11-03T15:37:40.324Z","logger":"raycluster-resource","msg":"default","name":"long-running-hvml9"}
{"level":"info","ts":"2026-11-03T15:37:40.324Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"long-running-hvml9","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:37:40.381Z","logger":"raycluster-resource","msg":"validate update","name":"long-running-hvml9"}
{"level":"info","ts":"2026-11-03T15:37:40.460Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"long-running-hvml9","reconcileID":"4f4b8e52-77bd-4724-8ff5-55c6f14e640f","cluster":"long-running-hvml9"}
{"level":"info","ts":"2026-11-03T15:37:40.460Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"long-running-hvml9","reconcileID":"7ea7bcc2-6a6a-4220-a851-4dea68cfb0c3","namespacedName":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:37:40.460Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"long-running-hvml9","reconcileID":"7ea7bcc2-6a6a-4220-a851-4dea68cfb0c3"}
{"level":"info","ts":"2026-11-03T15:37:40.460Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"long-running-hvml9","reconcileID":"7ea7bcc2-6a6a-4220-a851-4dea68cfb0c3","namespace":"test-ns-kbx4w","authClusters":1}
{"level":"info","ts":"2026-11-03T15:37:40.460Z","logger":"authentication-controller","msg":"ReferenceGrant not orphaned (clusters with auth exist)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"long-running-hvml9","reconcileID":"7ea7bcc2-6a6a-4220-a851-4dea68cfb0c3","namespace":"test-ns-kbx4w","authClusters":1}
{"level":"info","ts":"2026-11-03T15:37:40.460Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"long-running-hvml9","reconcileID":"af66a29b-c637-4e62-9cd8-c77601c1fc85","namespacedName":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:37:40.460Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"long-running-hvml9","reconcileID":"af66a29b-c637-4e62-9cd8-c77601c1fc85"}
{"level":"info","ts":"2026-11-03T15:37:40.460Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"cb1474e8-62c5-41aa-b2ff-6952057d0ddb","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:37:40.460Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"long-running-hvml9","reconcileID":"af66a29b-c637-4e62-9cd8-c77601c1fc85","namespace":"test-ns-kbx4w","authClusters":1}
{"level":"info","ts":"2026-11-03T15:37:40.460Z","logger":"authentication-controller","msg":"ReferenceGrant not orphaned (clusters with auth exist)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"long-running-hvml9","reconcileID":"af66a29b-c637-4e62-9cd8-c77601c1fc85","namespace":"test-ns-kbx4w","authClusters":1}
{"level":"info","ts":"2026-11-03T15:37:40.460Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"cb1474e8-62c5-41aa-b2ff-6952057d0ddb","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Failed","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":600,"Status.endTime":"2026-11-03 15:27:38 +0000 UTC","Now":"2026-11-03T15:37:40.460Z","ShutdownTime":"2026-11-03T15:37:38.000Z"}
{"level":"info","ts":"2026-11-03T15:37:40.460Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"cb1474e8-62c5-41aa-b2ff-6952057d0ddb","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:37:40.460Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"cb1474e8-62c5-41aa-b2ff-6952057d0ddb","isClusterDeleted":true}
{"level":"info","ts":"2026-11-03T15:37:40.460Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"cb1474e8-62c5-41aa-b2ff-6952057d0ddb","deletionMechanism":"ShutdownAfterJobFinishes","RayCluster":"long-running-hvml9"}
{"level":"info","ts":"2026-11-03T15:37:40.483Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"long-running-hvml9","reconcileID":"61fbccf6-ae5a-4277-803c-0e4a57c7f8cb","namespacedName":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:37:40.483Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"long-running-hvml9","reconcileID":"61fbccf6-ae5a-4277-803c-0e4a57c7f8cb"}
{"level":"info","ts":"2026-11-03T15:37:40.483Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"long-running-hvml9","reconcileID":"61fbccf6-ae5a-4277-803c-0e4a57c7f8cb","namespace":"test-ns-kbx4w","authClusters":1}
{"level":"info","ts":"2026-11-03T15:37:40.483Z","logger":"authentication-controller","msg":"ReferenceGrant not orphaned (clusters with auth exist)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"long-running-hvml9","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"long-running-hvml9","reconcileID":"61fbccf6-ae5a-4277-803c-0e4a57c7f8cb","namespace":"test-ns-kbx4w","authClusters":1}
{"level":"info","ts":"2026-11-03T15:37:41.591Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"26c365ac-44af-40bd-9828-19d533f962d2","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:37:41.591Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"26c365ac-44af-40bd-9828-19d533f962d2","name":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:37:41.591Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"26c365ac-44af-40bd-9828-19d533f962d2","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:37:41.592Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"26c365ac-44af-40bd-9828-19d533f962d2"}
{"level":"info","ts":"2026-11-03T15:37:41.592Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"26c365ac-44af-40bd-9828-19d533f962d2","RayCluster":"counter-auth-qmw8r","State":""}
{"level":"info","ts":"2026-11-03T15:37:41.592Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"26c365ac-44af-40bd-9828-19d533f962d2","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:37:44.592Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"05696f98-cdba-4c23-be2f-180e3745a739","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:37:44.592Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"05696f98-cdba-4c23-be2f-180e3745a739","name":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:37:44.592Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"05696f98-cdba-4c23-be2f-180e3745a739","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:37:44.593Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"05696f98-cdba-4c23-be2f-180e3745a739"}
{"level":"info","ts":"2026-11-03T15:37:44.593Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"05696f98-cdba-4c23-be2f-180e3745a739","RayCluster":"counter-auth-qmw8r","State":""}
{"level":"info","ts":"2026-11-03T15:37:44.593Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"05696f98-cdba-4c23-be2f-180e3745a739","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:37:47.594Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"ba9baf5a-2f51-4c73-8985-35af87c957cf","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:37:47.594Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"ba9baf5a-2f51-4c73-8985-35af87c957cf","name":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:37:47.594Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"ba9baf5a-2f51-4c73-8985-35af87c957cf","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:37:47.594Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"ba9baf5a-2f51-4c73-8985-35af87c957cf"}
{"level":"info","ts":"2026-11-03T15:37:47.594Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"ba9baf5a-2f51-4c73-8985-35af87c957cf","RayCluster":"counter-auth-qmw8r","State":""}
{"level":"info","ts":"2026-11-03T15:37:47.594Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"ba9baf5a-2f51-4c73-8985-35af87c957cf","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:37:50.595Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"2a006bbb-83de-4c7e-b019-d92e0b153d21","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:37:50.595Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"2a006bbb-83de-4c7e-b019-d92e0b153d21","name":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:37:50.595Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"2a006bbb-83de-4c7e-b019-d92e0b153d21","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:37:50.595Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"2a006bbb-83de-4c7e-b019-d92e0b153d21"}
{"level":"info","ts":"2026-11-03T15:37:50.595Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"2a006bbb-83de-4c7e-b019-d92e0b153d21","RayCluster":"counter-auth-qmw8r","State":""}
{"level":"info","ts":"2026-11-03T15:37:50.595Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"2a006bbb-83de-4c7e-b019-d92e0b153d21","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:37:53.597Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"dd5040d4-20b6-4d79-9f69-08878b4cfefe","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:37:53.597Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"dd5040d4-20b6-4d79-9f69-08878b4cfefe","name":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:37:53.597Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"dd5040d4-20b6-4d79-9f69-08878b4cfefe","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:37:53.597Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"dd5040d4-20b6-4d79-9f69-08878b4cfefe"}
{"level":"info","ts":"2026-11-03T15:37:53.597Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"dd5040d4-20b6-4d79-9f69-08878b4cfefe","RayCluster":"counter-auth-qmw8r","State":""}
{"level":"info","ts":"2026-11-03T15:37:53.597Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"dd5040d4-20b6-4d79-9f69-08878b4cfefe","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:37:56.598Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"591a5d31-bab7-4aae-9261-3b8f6210f8d2","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:37:56.598Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"591a5d31-bab7-4aae-9261-3b8f6210f8d2","name":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:37:56.598Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"591a5d31-bab7-4aae-9261-3b8f6210f8d2","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:37:56.598Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"591a5d31-bab7-4aae-9261-3b8f6210f8d2"}
{"level":"info","ts":"2026-11-03T15:37:56.598Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"591a5d31-bab7-4aae-9261-3b8f6210f8d2","RayCluster":"counter-auth-qmw8r","State":""}
{"level":"info","ts":"2026-11-03T15:37:56.598Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"591a5d31-bab7-4aae-9261-3b8f6210f8d2","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:37:59.598Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"66bf0a2f-d843-4653-b061-76dd05fe3d49","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:37:59.598Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"66bf0a2f-d843-4653-b061-76dd05fe3d49","name":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:37:59.598Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"66bf0a2f-d843-4653-b061-76dd05fe3d49","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:37:59.598Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"66bf0a2f-d843-4653-b061-76dd05fe3d49"}
{"level":"info","ts":"2026-11-03T15:37:59.598Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"66bf0a2f-d843-4653-b061-76dd05fe3d49","RayCluster":"counter-auth-qmw8r","State":""}
{"level":"info","ts":"2026-11-03T15:37:59.599Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"66bf0a2f-d843-4653-b061-76dd05fe3d49","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:38:00.925Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"6414a9db-3a87-4530-a6bb-a4db341b5617"}
{"level":"info","ts":"2026-11-03T15:38:00.925Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"6414a9db-3a87-4530-a6bb-a4db341b5617"}
{"level":"info","ts":"2026-11-03T15:38:00.925Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"6414a9db-3a87-4530-a6bb-a4db341b5617","1 head service found":"counter-auth-qmw8r-head-svc"}
{"level":"info","ts":"2026-11-03T15:38:00.925Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"6414a9db-3a87-4530-a6bb-a4db341b5617","Found 1 head Pod":"counter-auth-qmw8r-head-b964f","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:38:00.925Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"6414a9db-3a87-4530-a6bb-a4db341b5617","head Pod":"counter-auth-qmw8r-head-b964f","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-qmw8r-head-b964f. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:38:00.925Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"6414a9db-3a87-4530-a6bb-a4db341b5617","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:38:00.925Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"6414a9db-3a87-4530-a6bb-a4db341b5617","worker Pod":"counter-auth-qmw8r-small-group-worker-lqsm8","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-qmw8r-small-group-worker-lqsm8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:38:00.925Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"6414a9db-3a87-4530-a6bb-a4db341b5617","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:38:00.925Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"6414a9db-3a87-4530-a6bb-a4db341b5617","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:38:00.925Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"6414a9db-3a87-4530-a6bb-a4db341b5617","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:38:00.925Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"6414a9db-3a87-4530-a6bb-a4db341b5617","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:38:00.925Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"6414a9db-3a87-4530-a6bb-a4db341b5617","podName":"counter-auth-qmw8r-head-b964f","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:38:00.925Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"6414a9db-3a87-4530-a6bb-a4db341b5617","podName":"counter-auth-qmw8r-head-b964f","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:38:00.925Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"6414a9db-3a87-4530-a6bb-a4db341b5617","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:38:00.925Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"6414a9db-3a87-4530-a6bb-a4db341b5617","seconds":300}
{"level":"info","ts":"2026-11-03T15:38:01.882Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"044f8761-f3ee-421c-b8ba-133cfb921fd0"}
{"level":"info","ts":"2026-11-03T15:38:01.882Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"044f8761-f3ee-421c-b8ba-133cfb921fd0"}
{"level":"info","ts":"2026-11-03T15:38:01.882Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"044f8761-f3ee-421c-b8ba-133cfb921fd0","1 head service found":"counter-auth-qmw8r-head-svc"}
{"level":"info","ts":"2026-11-03T15:38:01.883Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"044f8761-f3ee-421c-b8ba-133cfb921fd0","Found 1 head Pod":"counter-auth-qmw8r-head-b964f","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:38:01.883Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"044f8761-f3ee-421c-b8ba-133cfb921fd0","head Pod":"counter-auth-qmw8r-head-b964f","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-qmw8r-head-b964f. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:38:01.883Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"044f8761-f3ee-421c-b8ba-133cfb921fd0","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:38:01.883Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"044f8761-f3ee-421c-b8ba-133cfb921fd0","worker Pod":"counter-auth-qmw8r-small-group-worker-lqsm8","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-qmw8r-small-group-worker-lqsm8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:38:01.883Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"044f8761-f3ee-421c-b8ba-133cfb921fd0","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:38:01.883Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"044f8761-f3ee-421c-b8ba-133cfb921fd0","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:38:01.883Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"044f8761-f3ee-421c-b8ba-133cfb921fd0","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:38:01.883Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"044f8761-f3ee-421c-b8ba-133cfb921fd0","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:38:01.883Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"044f8761-f3ee-421c-b8ba-133cfb921fd0","podName":"counter-auth-qmw8r-head-b964f","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:36:34Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:38:01.883Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"044f8761-f3ee-421c-b8ba-133cfb921fd0","podName":"counter-auth-qmw8r-head-b964f","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:36:34Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:38:01.883Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"044f8761-f3ee-421c-b8ba-133cfb921fd0","name":"counter-auth-qmw8r","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:01Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:38:01.947Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"d5ab851b-b201-46b7-aeb6-8442dce9072f","cluster":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:38:01.947Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"46c3230d-e35c-41ec-adf0-79929ef565f6","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:38:01.947Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"46c3230d-e35c-41ec-adf0-79929ef565f6","name":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:38:01.947Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"46c3230d-e35c-41ec-adf0-79929ef565f6","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:38:01.947Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"46c3230d-e35c-41ec-adf0-79929ef565f6"}
{"level":"info","ts":"2026-11-03T15:38:01.947Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"46c3230d-e35c-41ec-adf0-79929ef565f6","RayCluster":"counter-auth-qmw8r","State":""}
{"level":"info","ts":"2026-11-03T15:38:01.947Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"46c3230d-e35c-41ec-adf0-79929ef565f6","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:36:35Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:01Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:38:01.947Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"46c3230d-e35c-41ec-adf0-79929ef565f6","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:38:02.009Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"b24707b8-6a2f-4a99-b2eb-bba42fae4fca","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:38:02.009Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"b24707b8-6a2f-4a99-b2eb-bba42fae4fca","name":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:38:02.009Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"b24707b8-6a2f-4a99-b2eb-bba42fae4fca","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:38:02.009Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"b24707b8-6a2f-4a99-b2eb-bba42fae4fca"}
{"level":"info","ts":"2026-11-03T15:38:02.009Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"b24707b8-6a2f-4a99-b2eb-bba42fae4fca","RayCluster":"counter-auth-qmw8r","State":""}
{"level":"info","ts":"2026-11-03T15:38:02.009Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"b24707b8-6a2f-4a99-b2eb-bba42fae4fca","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:01Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:01Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:38:03.948Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"822848aa-5158-47d2-a577-46a3f645654a"}
{"level":"info","ts":"2026-11-03T15:38:03.948Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"822848aa-5158-47d2-a577-46a3f645654a"}
{"level":"info","ts":"2026-11-03T15:38:03.948Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"822848aa-5158-47d2-a577-46a3f645654a","1 head service found":"counter-auth-qmw8r-head-svc"}
{"level":"info","ts":"2026-11-03T15:38:03.948Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"822848aa-5158-47d2-a577-46a3f645654a","Found 1 head Pod":"counter-auth-qmw8r-head-b964f","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:38:03.948Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"822848aa-5158-47d2-a577-46a3f645654a","head Pod":"counter-auth-qmw8r-head-b964f","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-qmw8r-head-b964f. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:38:03.948Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"822848aa-5158-47d2-a577-46a3f645654a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:38:03.948Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"822848aa-5158-47d2-a577-46a3f645654a","worker Pod":"counter-auth-qmw8r-small-group-worker-lqsm8","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-qmw8r-small-group-worker-lqsm8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:38:03.948Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"822848aa-5158-47d2-a577-46a3f645654a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:38:03.948Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"822848aa-5158-47d2-a577-46a3f645654a","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:38:03.948Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"822848aa-5158-47d2-a577-46a3f645654a","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:38:03.948Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"822848aa-5158-47d2-a577-46a3f645654a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:38:03.948Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"822848aa-5158-47d2-a577-46a3f645654a","podName":"counter-auth-qmw8r-small-group-worker-lqsm8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:38:03.948Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"822848aa-5158-47d2-a577-46a3f645654a","podName":"counter-auth-qmw8r-small-group-worker-lqsm8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:38:03.948Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"822848aa-5158-47d2-a577-46a3f645654a","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:38:03.948Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"822848aa-5158-47d2-a577-46a3f645654a","seconds":300}
{"level":"info","ts":"2026-11-03T15:38:05.010Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"8c3e6e38-4514-4197-94a0-f4233f073d54","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:38:05.010Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"8c3e6e38-4514-4197-94a0-f4233f073d54","name":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:38:05.010Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"8c3e6e38-4514-4197-94a0-f4233f073d54","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:38:05.011Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"8c3e6e38-4514-4197-94a0-f4233f073d54"}
{"level":"info","ts":"2026-11-03T15:38:05.011Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"8c3e6e38-4514-4197-94a0-f4233f073d54","RayCluster":"counter-auth-qmw8r","State":""}
{"level":"info","ts":"2026-11-03T15:38:05.011Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"8c3e6e38-4514-4197-94a0-f4233f073d54","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:01Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:01Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:38:06.674Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"c25a7f2b-282a-4ba2-ab7d-c54861969438"}
{"level":"info","ts":"2026-11-03T15:38:06.674Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"c25a7f2b-282a-4ba2-ab7d-c54861969438"}
{"level":"info","ts":"2026-11-03T15:38:06.674Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"c25a7f2b-282a-4ba2-ab7d-c54861969438","1 head service found":"counter-auth-qmw8r-head-svc"}
{"level":"info","ts":"2026-11-03T15:38:06.674Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"c25a7f2b-282a-4ba2-ab7d-c54861969438","Found 1 head Pod":"counter-auth-qmw8r-head-b964f","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:38:06.674Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"c25a7f2b-282a-4ba2-ab7d-c54861969438","head Pod":"counter-auth-qmw8r-head-b964f","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-qmw8r-head-b964f. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:38:06.674Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"c25a7f2b-282a-4ba2-ab7d-c54861969438","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:38:06.674Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"c25a7f2b-282a-4ba2-ab7d-c54861969438","worker Pod":"counter-auth-qmw8r-small-group-worker-lqsm8","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-qmw8r-small-group-worker-lqsm8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:38:06.674Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"c25a7f2b-282a-4ba2-ab7d-c54861969438","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:38:06.674Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"c25a7f2b-282a-4ba2-ab7d-c54861969438","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:38:06.674Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"c25a7f2b-282a-4ba2-ab7d-c54861969438","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:38:06.674Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"c25a7f2b-282a-4ba2-ab7d-c54861969438","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:38:06.675Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"c25a7f2b-282a-4ba2-ab7d-c54861969438","podName":"counter-auth-qmw8r-head-b964f","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:36:34Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:38:06.675Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"c25a7f2b-282a-4ba2-ab7d-c54861969438","podName":"counter-auth-qmw8r-head-b964f","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:36:34Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:38:06.675Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"c25a7f2b-282a-4ba2-ab7d-c54861969438","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:38:06.675Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"c25a7f2b-282a-4ba2-ab7d-c54861969438","seconds":300}
{"level":"info","ts":"2026-11-03T15:38:07.677Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"5c471b82-0bbe-4fd0-bd38-e82629d8187e"}
{"level":"info","ts":"2026-11-03T15:38:07.677Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"5c471b82-0bbe-4fd0-bd38-e82629d8187e"}
{"level":"info","ts":"2026-11-03T15:38:07.677Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"5c471b82-0bbe-4fd0-bd38-e82629d8187e","1 head service found":"counter-auth-qmw8r-head-svc"}
{"level":"info","ts":"2026-11-03T15:38:07.678Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"5c471b82-0bbe-4fd0-bd38-e82629d8187e","Found 1 head Pod":"counter-auth-qmw8r-head-b964f","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:38:07.678Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"5c471b82-0bbe-4fd0-bd38-e82629d8187e","head Pod":"counter-auth-qmw8r-head-b964f","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-qmw8r-head-b964f. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:38:07.678Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"5c471b82-0bbe-4fd0-bd38-e82629d8187e","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:38:07.678Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"5c471b82-0bbe-4fd0-bd38-e82629d8187e","worker Pod":"counter-auth-qmw8r-small-group-worker-lqsm8","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-qmw8r-small-group-worker-lqsm8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:38:07.678Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"5c471b82-0bbe-4fd0-bd38-e82629d8187e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:38:07.678Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"5c471b82-0bbe-4fd0-bd38-e82629d8187e","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:38:07.678Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"5c471b82-0bbe-4fd0-bd38-e82629d8187e","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:38:07.678Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"5c471b82-0bbe-4fd0-bd38-e82629d8187e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:38:07.678Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"5c471b82-0bbe-4fd0-bd38-e82629d8187e","podName":"counter-auth-qmw8r-head-b964f","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:36:34Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:38:07.678Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"5c471b82-0bbe-4fd0-bd38-e82629d8187e","podName":"counter-auth-qmw8r-head-b964f","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:36:34Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:38:07.679Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"5c471b82-0bbe-4fd0-bd38-e82629d8187e","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:38:07.679Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"5c471b82-0bbe-4fd0-bd38-e82629d8187e","seconds":300}
{"level":"info","ts":"2026-11-03T15:38:08.011Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"0c09a319-c3f6-4553-8ff3-4d23e7d64b19","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:38:08.011Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"0c09a319-c3f6-4553-8ff3-4d23e7d64b19","name":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:38:08.011Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"0c09a319-c3f6-4553-8ff3-4d23e7d64b19","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:38:08.012Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"0c09a319-c3f6-4553-8ff3-4d23e7d64b19"}
{"level":"info","ts":"2026-11-03T15:38:08.012Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"0c09a319-c3f6-4553-8ff3-4d23e7d64b19","RayCluster":"counter-auth-qmw8r","State":""}
{"level":"info","ts":"2026-11-03T15:38:08.012Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"0c09a319-c3f6-4553-8ff3-4d23e7d64b19","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:01Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:01Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:38:11.013Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"4fc86782-59aa-4db9-b32f-21f85dd71fa7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:38:11.013Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"4fc86782-59aa-4db9-b32f-21f85dd71fa7","name":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:38:11.013Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"4fc86782-59aa-4db9-b32f-21f85dd71fa7","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:38:11.013Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"4fc86782-59aa-4db9-b32f-21f85dd71fa7"}
{"level":"info","ts":"2026-11-03T15:38:11.013Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"4fc86782-59aa-4db9-b32f-21f85dd71fa7","RayCluster":"counter-auth-qmw8r","State":""}
{"level":"info","ts":"2026-11-03T15:38:11.013Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"4fc86782-59aa-4db9-b32f-21f85dd71fa7","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:01Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:01Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:38:12.941Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"2b1f680e-515b-4604-8b97-9364b455e87d"}
{"level":"info","ts":"2026-11-03T15:38:12.941Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"2b1f680e-515b-4604-8b97-9364b455e87d"}
{"level":"info","ts":"2026-11-03T15:38:12.941Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"2b1f680e-515b-4604-8b97-9364b455e87d","1 head service found":"counter-auth-qmw8r-head-svc"}
{"level":"info","ts":"2026-11-03T15:38:12.941Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"2b1f680e-515b-4604-8b97-9364b455e87d","Found 1 head Pod":"counter-auth-qmw8r-head-b964f","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:38:12.941Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"2b1f680e-515b-4604-8b97-9364b455e87d","head Pod":"counter-auth-qmw8r-head-b964f","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-qmw8r-head-b964f. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:38:12.941Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"2b1f680e-515b-4604-8b97-9364b455e87d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:38:12.941Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"2b1f680e-515b-4604-8b97-9364b455e87d","worker Pod":"counter-auth-qmw8r-small-group-worker-lqsm8","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-qmw8r-small-group-worker-lqsm8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:38:12.941Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"2b1f680e-515b-4604-8b97-9364b455e87d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:38:12.941Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"2b1f680e-515b-4604-8b97-9364b455e87d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:38:12.941Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"2b1f680e-515b-4604-8b97-9364b455e87d","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:38:12.941Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"2b1f680e-515b-4604-8b97-9364b455e87d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:38:12.941Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"2b1f680e-515b-4604-8b97-9364b455e87d","podName":"counter-auth-qmw8r-small-group-worker-lqsm8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:38:12.941Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"2b1f680e-515b-4604-8b97-9364b455e87d","podName":"counter-auth-qmw8r-small-group-worker-lqsm8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:38:12.941Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"2b1f680e-515b-4604-8b97-9364b455e87d","name":"counter-auth-qmw8r","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:01Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:12Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:38:12Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:38:13.002Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"a8aa8c92-9e1a-44fb-b576-d3c3992c70d8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:38:13.003Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"a8aa8c92-9e1a-44fb-b576-d3c3992c70d8","name":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:38:13.003Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"a8aa8c92-9e1a-44fb-b576-d3c3992c70d8","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:38:13.002Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"c71fb7cc-addd-4247-b467-a9a0d94f51ab","cluster":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:38:13.003Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"a8aa8c92-9e1a-44fb-b576-d3c3992c70d8"}
{"level":"info","ts":"2026-11-03T15:38:13.003Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"a8aa8c92-9e1a-44fb-b576-d3c3992c70d8","RayCluster":"counter-auth-qmw8r","State":""}
{"level":"info","ts":"2026-11-03T15:38:13.003Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"a8aa8c92-9e1a-44fb-b576-d3c3992c70d8","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:01Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:12Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:38:12Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:38:13.003Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"a8aa8c92-9e1a-44fb-b576-d3c3992c70d8","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:38:13.064Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"7a4e0d55-af80-460e-877e-a660b635addf","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:38:13.064Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"7a4e0d55-af80-460e-877e-a660b635addf","name":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:38:13.064Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"7a4e0d55-af80-460e-877e-a660b635addf","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:38:13.064Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"7a4e0d55-af80-460e-877e-a660b635addf"}
{"level":"info","ts":"2026-11-03T15:38:13.064Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"7a4e0d55-af80-460e-877e-a660b635addf","RayCluster":"counter-auth-qmw8r","State":""}
{"level":"info","ts":"2026-11-03T15:38:13.065Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"7a4e0d55-af80-460e-877e-a660b635addf","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:12Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:38:12Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:12Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:38:12Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:38:15.003Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"a3591c02-482b-4786-bad6-542538edbd50"}
{"level":"info","ts":"2026-11-03T15:38:15.003Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"a3591c02-482b-4786-bad6-542538edbd50"}
{"level":"info","ts":"2026-11-03T15:38:15.003Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"a3591c02-482b-4786-bad6-542538edbd50","1 head service found":"counter-auth-qmw8r-head-svc"}
{"level":"info","ts":"2026-11-03T15:38:15.003Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"a3591c02-482b-4786-bad6-542538edbd50","Found 1 head Pod":"counter-auth-qmw8r-head-b964f","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:38:15.003Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"a3591c02-482b-4786-bad6-542538edbd50","head Pod":"counter-auth-qmw8r-head-b964f","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-qmw8r-head-b964f. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:38:15.003Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"a3591c02-482b-4786-bad6-542538edbd50","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:38:15.003Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"a3591c02-482b-4786-bad6-542538edbd50","worker Pod":"counter-auth-qmw8r-small-group-worker-lqsm8","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-qmw8r-small-group-worker-lqsm8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:38:15.003Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"a3591c02-482b-4786-bad6-542538edbd50","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:38:15.003Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"a3591c02-482b-4786-bad6-542538edbd50","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:38:15.004Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"a3591c02-482b-4786-bad6-542538edbd50","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:38:15.004Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"a3591c02-482b-4786-bad6-542538edbd50","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:38:15.004Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"a3591c02-482b-4786-bad6-542538edbd50","podName":"counter-auth-qmw8r-small-group-worker-lqsm8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:38:15.004Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"a3591c02-482b-4786-bad6-542538edbd50","podName":"counter-auth-qmw8r-small-group-worker-lqsm8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:38:15.004Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"a3591c02-482b-4786-bad6-542538edbd50","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:38:15.004Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"a3591c02-482b-4786-bad6-542538edbd50","seconds":300}
{"level":"info","ts":"2026-11-03T15:38:16.065Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"a4a069cc-58cf-47d3-9405-a161de6ec60b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:38:16.065Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"a4a069cc-58cf-47d3-9405-a161de6ec60b","name":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:38:16.065Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"a4a069cc-58cf-47d3-9405-a161de6ec60b","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:38:16.066Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"a4a069cc-58cf-47d3-9405-a161de6ec60b"}
{"level":"info","ts":"2026-11-03T15:38:16.066Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"a4a069cc-58cf-47d3-9405-a161de6ec60b","RayCluster":"counter-auth-qmw8r","State":""}
{"level":"info","ts":"2026-11-03T15:38:16.066Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"a4a069cc-58cf-47d3-9405-a161de6ec60b","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:12Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:38:12Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:12Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:38:12Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:38:17.716Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"2b2952b0-c682-4807-93d7-1f28392662e3"}
{"level":"info","ts":"2026-11-03T15:38:17.716Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"2b2952b0-c682-4807-93d7-1f28392662e3"}
{"level":"info","ts":"2026-11-03T15:38:17.716Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"2b2952b0-c682-4807-93d7-1f28392662e3","1 head service found":"counter-auth-qmw8r-head-svc"}
{"level":"info","ts":"2026-11-03T15:38:17.716Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"2b2952b0-c682-4807-93d7-1f28392662e3","Found 1 head Pod":"counter-auth-qmw8r-head-b964f","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:38:17.716Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"2b2952b0-c682-4807-93d7-1f28392662e3","head Pod":"counter-auth-qmw8r-head-b964f","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-qmw8r-head-b964f. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:38:17.716Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"2b2952b0-c682-4807-93d7-1f28392662e3","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:38:17.716Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"2b2952b0-c682-4807-93d7-1f28392662e3","worker Pod":"counter-auth-qmw8r-small-group-worker-lqsm8","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-qmw8r-small-group-worker-lqsm8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:38:17.716Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"2b2952b0-c682-4807-93d7-1f28392662e3","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:38:17.716Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"2b2952b0-c682-4807-93d7-1f28392662e3","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:38:17.716Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"2b2952b0-c682-4807-93d7-1f28392662e3","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:38:17.716Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"2b2952b0-c682-4807-93d7-1f28392662e3","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:38:17.716Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"2b2952b0-c682-4807-93d7-1f28392662e3","podName":"counter-auth-qmw8r-small-group-worker-lqsm8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:38:17.716Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"2b2952b0-c682-4807-93d7-1f28392662e3","podName":"counter-auth-qmw8r-small-group-worker-lqsm8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:38:17.716Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"2b2952b0-c682-4807-93d7-1f28392662e3","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:38:17.716Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"2b2952b0-c682-4807-93d7-1f28392662e3","seconds":300}
{"level":"info","ts":"2026-11-03T15:38:18.725Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"695f5ad6-693c-47d4-836c-0d7285ea4cfd"}
{"level":"info","ts":"2026-11-03T15:38:18.725Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"695f5ad6-693c-47d4-836c-0d7285ea4cfd"}
{"level":"info","ts":"2026-11-03T15:38:18.725Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"695f5ad6-693c-47d4-836c-0d7285ea4cfd","1 head service found":"counter-auth-qmw8r-head-svc"}
{"level":"info","ts":"2026-11-03T15:38:18.725Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"695f5ad6-693c-47d4-836c-0d7285ea4cfd","Found 1 head Pod":"counter-auth-qmw8r-head-b964f","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:38:18.725Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"695f5ad6-693c-47d4-836c-0d7285ea4cfd","head Pod":"counter-auth-qmw8r-head-b964f","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-qmw8r-head-b964f. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:38:18.725Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"695f5ad6-693c-47d4-836c-0d7285ea4cfd","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:38:18.725Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"695f5ad6-693c-47d4-836c-0d7285ea4cfd","worker Pod":"counter-auth-qmw8r-small-group-worker-lqsm8","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-qmw8r-small-group-worker-lqsm8. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:38:18.725Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"695f5ad6-693c-47d4-836c-0d7285ea4cfd","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:38:18.725Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"695f5ad6-693c-47d4-836c-0d7285ea4cfd","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:38:18.725Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"695f5ad6-693c-47d4-836c-0d7285ea4cfd","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:38:18.725Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"695f5ad6-693c-47d4-836c-0d7285ea4cfd","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:38:18.725Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"695f5ad6-693c-47d4-836c-0d7285ea4cfd","podName":"counter-auth-qmw8r-small-group-worker-lqsm8","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:36:34Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:38:18.725Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"695f5ad6-693c-47d4-836c-0d7285ea4cfd","podName":"counter-auth-qmw8r-small-group-worker-lqsm8","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:36:34Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:38:18.725Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"695f5ad6-693c-47d4-836c-0d7285ea4cfd","name":"counter-auth-qmw8r","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:12Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:38:12Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:18Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:38:12Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:38:18.786Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"9bdd3d01-9ddd-40c6-aa22-9e4b26b75e75","cluster":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:38:18.786Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"38be0688-75f8-4b73-b6c4-481197b17b04","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:38:18.786Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"38be0688-75f8-4b73-b6c4-481197b17b04","name":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:38:18.786Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"38be0688-75f8-4b73-b6c4-481197b17b04","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:38:18.786Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"38be0688-75f8-4b73-b6c4-481197b17b04"}
{"level":"info","ts":"2026-11-03T15:38:18.786Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"38be0688-75f8-4b73-b6c4-481197b17b04","RayCluster":"counter-auth-qmw8r","State":""}
{"level":"info","ts":"2026-11-03T15:38:18.786Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"38be0688-75f8-4b73-b6c4-481197b17b04","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:12Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:38:12Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:18Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:38:12Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:38:18.786Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"38be0688-75f8-4b73-b6c4-481197b17b04","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:38:18.848Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"a8a08180-3183-4ab2-83f9-fbe7ed16e11c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:38:18.848Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"a8a08180-3183-4ab2-83f9-fbe7ed16e11c","name":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:38:18.848Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"a8a08180-3183-4ab2-83f9-fbe7ed16e11c","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:38:18.848Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"a8a08180-3183-4ab2-83f9-fbe7ed16e11c"}
{"level":"info","ts":"2026-11-03T15:38:18.848Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"a8a08180-3183-4ab2-83f9-fbe7ed16e11c","RayCluster":"counter-auth-qmw8r","State":""}
{"level":"info","ts":"2026-11-03T15:38:18.848Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"a8a08180-3183-4ab2-83f9-fbe7ed16e11c","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:18Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:38:12Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:18Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:38:12Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:38:20.786Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"510e65e4-bc8f-4c42-8ab4-7b090f61fd20"}
{"level":"info","ts":"2026-11-03T15:38:20.786Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"510e65e4-bc8f-4c42-8ab4-7b090f61fd20"}
{"level":"info","ts":"2026-11-03T15:38:20.786Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"510e65e4-bc8f-4c42-8ab4-7b090f61fd20","1 head service found":"counter-auth-qmw8r-head-svc"}
{"level":"info","ts":"2026-11-03T15:38:20.786Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"510e65e4-bc8f-4c42-8ab4-7b090f61fd20","Found 1 head Pod":"counter-auth-qmw8r-head-b964f","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:38:20.786Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"510e65e4-bc8f-4c42-8ab4-7b090f61fd20","head Pod":"counter-auth-qmw8r-head-b964f","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-qmw8r-head-b964f. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:38:20.786Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"510e65e4-bc8f-4c42-8ab4-7b090f61fd20","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:38:20.786Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"510e65e4-bc8f-4c42-8ab4-7b090f61fd20","worker Pod":"counter-auth-qmw8r-small-group-worker-lqsm8","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-qmw8r-small-group-worker-lqsm8. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:38:20.786Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"510e65e4-bc8f-4c42-8ab4-7b090f61fd20","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:38:20.786Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"510e65e4-bc8f-4c42-8ab4-7b090f61fd20","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:38:20.786Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"510e65e4-bc8f-4c42-8ab4-7b090f61fd20","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:38:20.786Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"510e65e4-bc8f-4c42-8ab4-7b090f61fd20","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:38:20.786Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"510e65e4-bc8f-4c42-8ab4-7b090f61fd20","podName":"counter-auth-qmw8r-small-group-worker-lqsm8","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:36:34Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:38:20.786Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"510e65e4-bc8f-4c42-8ab4-7b090f61fd20","podName":"counter-auth-qmw8r-small-group-worker-lqsm8","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:36:34Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:38:20.786Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"510e65e4-bc8f-4c42-8ab4-7b090f61fd20","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:38:20.786Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"510e65e4-bc8f-4c42-8ab4-7b090f61fd20","seconds":300}
{"level":"info","ts":"2026-11-03T15:38:21.849Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"28709f2a-d4f2-4f75-9dfd-d9d6366e9b6d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:38:21.849Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"28709f2a-d4f2-4f75-9dfd-d9d6366e9b6d","name":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:38:21.849Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"28709f2a-d4f2-4f75-9dfd-d9d6366e9b6d","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:38:21.849Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"28709f2a-d4f2-4f75-9dfd-d9d6366e9b6d"}
{"level":"info","ts":"2026-11-03T15:38:21.849Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"28709f2a-d4f2-4f75-9dfd-d9d6366e9b6d","RayCluster":"counter-auth-qmw8r","State":""}
{"level":"info","ts":"2026-11-03T15:38:21.849Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"28709f2a-d4f2-4f75-9dfd-d9d6366e9b6d","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:18Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:38:12Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:18Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:38:12Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:38:24.850Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"f6192fa0-654b-4790-8a9f-07f303b84492","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:38:24.850Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"f6192fa0-654b-4790-8a9f-07f303b84492","name":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:38:24.850Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"f6192fa0-654b-4790-8a9f-07f303b84492","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:38:24.850Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"f6192fa0-654b-4790-8a9f-07f303b84492"}
{"level":"info","ts":"2026-11-03T15:38:24.850Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"f6192fa0-654b-4790-8a9f-07f303b84492","RayCluster":"counter-auth-qmw8r","State":""}
{"level":"info","ts":"2026-11-03T15:38:24.850Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"f6192fa0-654b-4790-8a9f-07f303b84492","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:18Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:38:12Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:18Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:38:12Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:38:27.850Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"ee8c8b3f-082e-4b8a-855f-a998f965181c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:38:27.850Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"ee8c8b3f-082e-4b8a-855f-a998f965181c","name":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:38:27.850Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"ee8c8b3f-082e-4b8a-855f-a998f965181c","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:38:27.851Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"ee8c8b3f-082e-4b8a-855f-a998f965181c"}
{"level":"info","ts":"2026-11-03T15:38:27.851Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"ee8c8b3f-082e-4b8a-855f-a998f965181c","RayCluster":"counter-auth-qmw8r","State":""}
{"level":"info","ts":"2026-11-03T15:38:27.851Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"ee8c8b3f-082e-4b8a-855f-a998f965181c","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:18Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:38:12Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:18Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:38:12Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:38:29.735Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"c9e3f8ee-62a4-4f6d-8a65-cba4369e3176"}
{"level":"info","ts":"2026-11-03T15:38:29.735Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"c9e3f8ee-62a4-4f6d-8a65-cba4369e3176"}
{"level":"info","ts":"2026-11-03T15:38:29.735Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"c9e3f8ee-62a4-4f6d-8a65-cba4369e3176","1 head service found":"counter-auth-qmw8r-head-svc"}
{"level":"info","ts":"2026-11-03T15:38:29.735Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"c9e3f8ee-62a4-4f6d-8a65-cba4369e3176","Found 1 head Pod":"counter-auth-qmw8r-head-b964f","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:38:29.735Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"c9e3f8ee-62a4-4f6d-8a65-cba4369e3176","head Pod":"counter-auth-qmw8r-head-b964f","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-qmw8r-head-b964f. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:38:29.735Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"c9e3f8ee-62a4-4f6d-8a65-cba4369e3176","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:38:29.735Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"c9e3f8ee-62a4-4f6d-8a65-cba4369e3176","worker Pod":"counter-auth-qmw8r-small-group-worker-lqsm8","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-qmw8r-small-group-worker-lqsm8. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:38:29.735Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"c9e3f8ee-62a4-4f6d-8a65-cba4369e3176","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:38:29.735Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"c9e3f8ee-62a4-4f6d-8a65-cba4369e3176","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:38:29.735Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"c9e3f8ee-62a4-4f6d-8a65-cba4369e3176","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:38:29.735Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"c9e3f8ee-62a4-4f6d-8a65-cba4369e3176","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:38:29.735Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"c9e3f8ee-62a4-4f6d-8a65-cba4369e3176","name":"counter-auth-qmw8r","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:18Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:38:12Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:29Z","stateTransitionTimes":{"ready":"2026-11-03T15:38:29Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:38:12Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:38:29Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:38:29.797Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"f9923f12-bd01-48ae-82cc-897ffdc3c0b1","cluster":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:38:29.797Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"ec1d8b93-67f6-467d-acb6-131336abfe9a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:38:29.797Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"ec1d8b93-67f6-467d-acb6-131336abfe9a","name":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:38:29.797Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"ec1d8b93-67f6-467d-acb6-131336abfe9a","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:38:29.797Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"ec1d8b93-67f6-467d-acb6-131336abfe9a"}
{"level":"info","ts":"2026-11-03T15:38:29.861Z","logger":"controllers.RayJob","msg":"Created submitter Kubernetes Job for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"ec1d8b93-67f6-467d-acb6-131336abfe9a","Kubernetes Job":"counter-auth"}
{"level":"info","ts":"2026-11-03T15:38:29.861Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"ec1d8b93-67f6-467d-acb6-131336abfe9a","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:18Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:38:12Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","dashboardURL":"counter-auth-qmw8r-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:18Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:38:12Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:38:29.861Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"ec1d8b93-67f6-467d-acb6-131336abfe9a","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-11-03T15:38:29.922Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"ea1a8a95-181d-484a-a494-f94857ad098f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:38:29.922Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"ea1a8a95-181d-484a-a494-f94857ad098f","name":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:38:29.922Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"ea1a8a95-181d-484a-a494-f94857ad098f","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:38:29.922Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"ea1a8a95-181d-484a-a494-f94857ad098f"}
{"level":"info","ts":"2026-11-03T15:38:29.922Z","logger":"controllers.RayJob","msg":"The submitter Kubernetes Job for RayJob already exists","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"ea1a8a95-181d-484a-a494-f94857ad098f","Kubernetes Job":"counter-auth"}
{"level":"info","ts":"2026-11-03T15:38:29.922Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"ea1a8a95-181d-484a-a494-f94857ad098f","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:18Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:38:12Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","dashboardURL":"counter-auth-qmw8r-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:18Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:38:12Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:38:29.922Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"ea1a8a95-181d-484a-a494-f94857ad098f","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-11-03T15:38:29.978Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"ea1a8a95-181d-484a-a494-f94857ad098f","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter-auth\": the object has been modified; please apply your changes to the latest version and try again"}
{"level":"info","ts":"2026-11-03T15:38:29.978Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"ea1a8a95-181d-484a-a494-f94857ad098f"}
{"level":"error","ts":"2026-11-03T15:38:29.978Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"ea1a8a95-181d-484a-a494-f94857ad098f","error":"Operation cannot be fulfilled on rayjobs.ray.io \"counter-auth\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:38:29.978Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"510ec3ca-8ea2-455b-b545-096fa4b22e03","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:38:29.978Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"510ec3ca-8ea2-455b-b545-096fa4b22e03","name":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:38:29.978Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"510ec3ca-8ea2-455b-b545-096fa4b22e03","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:38:29.978Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"510ec3ca-8ea2-455b-b545-096fa4b22e03"}
{"level":"info","ts":"2026-11-03T15:38:29.978Z","logger":"controllers.RayJob","msg":"Create a Kubernetes Service","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"510ec3ca-8ea2-455b-b545-096fa4b22e03","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:38:30.043Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"d8608287-3bff-493f-a104-96a7162da3b9","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:38:30.043Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"d8608287-3bff-493f-a104-96a7162da3b9","name":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:38:30.043Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"d8608287-3bff-493f-a104-96a7162da3b9","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:38:30.043Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"d8608287-3bff-493f-a104-96a7162da3b9"}
{"level":"info","ts":"2026-11-03T15:38:30.044Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"d8608287-3bff-493f-a104-96a7162da3b9","rayCluster":"counter-auth-qmw8r","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:38:31.798Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"1e2cf55e-4eed-4623-81f6-3d27872f0cab"}
{"level":"info","ts":"2026-11-03T15:38:31.798Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"1e2cf55e-4eed-4623-81f6-3d27872f0cab"}
{"level":"info","ts":"2026-11-03T15:38:31.798Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"1e2cf55e-4eed-4623-81f6-3d27872f0cab","1 head service found":"counter-auth-qmw8r-head-svc"}
{"level":"info","ts":"2026-11-03T15:38:31.798Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"1e2cf55e-4eed-4623-81f6-3d27872f0cab","Found 1 head Pod":"counter-auth-qmw8r-head-b964f","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:38:31.798Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"1e2cf55e-4eed-4623-81f6-3d27872f0cab","head Pod":"counter-auth-qmw8r-head-b964f","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod counter-auth-qmw8r-head-b964f. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:38:31.798Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"1e2cf55e-4eed-4623-81f6-3d27872f0cab","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:38:31.798Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"1e2cf55e-4eed-4623-81f6-3d27872f0cab","worker Pod":"counter-auth-qmw8r-small-group-worker-lqsm8","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod counter-auth-qmw8r-small-group-worker-lqsm8. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:38:31.798Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"1e2cf55e-4eed-4623-81f6-3d27872f0cab","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:38:31.798Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"1e2cf55e-4eed-4623-81f6-3d27872f0cab","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:38:31.798Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"1e2cf55e-4eed-4623-81f6-3d27872f0cab","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:38:31.798Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"1e2cf55e-4eed-4623-81f6-3d27872f0cab","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:38:31.798Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"1e2cf55e-4eed-4623-81f6-3d27872f0cab","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:38:31.798Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"1e2cf55e-4eed-4623-81f6-3d27872f0cab","seconds":300}
{"level":"info","ts":"2026-11-03T15:38:32.688Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"2623ae8c-65e7-46b0-98a5-355619fd4dd5","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:38:32.688Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"2623ae8c-65e7-46b0-98a5-355619fd4dd5","name":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:38:32.688Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"2623ae8c-65e7-46b0-98a5-355619fd4dd5","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:38:32.689Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"2623ae8c-65e7-46b0-98a5-355619fd4dd5"}
{"level":"info","ts":"2026-11-03T15:38:32.689Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"2623ae8c-65e7-46b0-98a5-355619fd4dd5","rayCluster":"counter-auth-qmw8r","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:38:35.695Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"44184067-b547-4fff-932e-41b70251315f","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:38:35.695Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"44184067-b547-4fff-932e-41b70251315f","name":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:38:35.695Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"44184067-b547-4fff-932e-41b70251315f","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:38:35.695Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"44184067-b547-4fff-932e-41b70251315f"}
{"level":"info","ts":"2026-11-03T15:38:35.695Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"44184067-b547-4fff-932e-41b70251315f","rayCluster":"counter-auth-qmw8r","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:38:38.701Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"c5e4c275-faa0-4f23-ac7e-93d2c6b904b4","JobStatus":"","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:38:38.701Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"c5e4c275-faa0-4f23-ac7e-93d2c6b904b4","name":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:38:38.702Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"c5e4c275-faa0-4f23-ac7e-93d2c6b904b4","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:38:38.702Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"c5e4c275-faa0-4f23-ac7e-93d2c6b904b4"}
{"level":"info","ts":"2026-11-03T15:38:38.702Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"c5e4c275-faa0-4f23-ac7e-93d2c6b904b4","rayCluster":"counter-auth-qmw8r","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:38:38.722Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"c5e4c275-faa0-4f23-ac7e-93d2c6b904b4","oldRayJobStatus":{"rayJobInfo":{},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","dashboardURL":"counter-auth-qmw8r-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobDeploymentStatus":"Running","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:18Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:38:12Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:38:38Z"},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","dashboardURL":"counter-auth-qmw8r-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:29Z","stateTransitionTimes":{"ready":"2026-11-03T15:38:29Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:38:12Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:38:29Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:38:38.722Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"c5e4c275-faa0-4f23-ac7e-93d2c6b904b4","old JobStatus":"","new JobStatus":"PENDING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-11-03T15:38:38.831Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"e1c24dc4-e301-4299-9251-134989dc417f","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:38:38.831Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"e1c24dc4-e301-4299-9251-134989dc417f","name":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:38:38.831Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"e1c24dc4-e301-4299-9251-134989dc417f","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:38:38.831Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"e1c24dc4-e301-4299-9251-134989dc417f"}
{"level":"info","ts":"2026-11-03T15:38:38.831Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"e1c24dc4-e301-4299-9251-134989dc417f","rayCluster":"counter-auth-qmw8r","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:38:38.834Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"e1c24dc4-e301-4299-9251-134989dc417f","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:38:38Z"},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","dashboardURL":"counter-auth-qmw8r-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:29Z","stateTransitionTimes":{"ready":"2026-11-03T15:38:29Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:38:12Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:38:29Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:38:38Z"},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","dashboardURL":"counter-auth-qmw8r-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:29Z","stateTransitionTimes":{"ready":"2026-11-03T15:38:29Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:38:12Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:38:29Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:38:41.835Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"d895afaa-5646-4a9d-b3bc-e28c24696c6b","JobStatus":"PENDING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:38:41.835Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"d895afaa-5646-4a9d-b3bc-e28c24696c6b","name":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:38:41.835Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"d895afaa-5646-4a9d-b3bc-e28c24696c6b","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:38:41.835Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"d895afaa-5646-4a9d-b3bc-e28c24696c6b"}
{"level":"info","ts":"2026-11-03T15:38:41.836Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"d895afaa-5646-4a9d-b3bc-e28c24696c6b","rayCluster":"counter-auth-qmw8r","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:38:41.839Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"d895afaa-5646-4a9d-b3bc-e28c24696c6b","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:38:38Z"},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","dashboardURL":"counter-auth-qmw8r-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"PENDING","jobDeploymentStatus":"Running","message":"Job has not started yet. It may be waiting for the runtime environment to be set up.","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:29Z","stateTransitionTimes":{"ready":"2026-11-03T15:38:29Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:38:12Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:38:29Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:38:38Z"},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","dashboardURL":"counter-auth-qmw8r-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:29Z","stateTransitionTimes":{"ready":"2026-11-03T15:38:29Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:38:12Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:38:29Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:38:41.839Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"d895afaa-5646-4a9d-b3bc-e28c24696c6b","old JobStatus":"PENDING","new JobStatus":"RUNNING","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-11-03T15:38:41.936Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"4c41ee65-915d-4acc-a7ef-ae4d93576cc8","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:38:41.936Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"4c41ee65-915d-4acc-a7ef-ae4d93576cc8","name":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:38:41.936Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"4c41ee65-915d-4acc-a7ef-ae4d93576cc8","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:38:41.936Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"4c41ee65-915d-4acc-a7ef-ae4d93576cc8"}
{"level":"info","ts":"2026-11-03T15:38:41.936Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"4c41ee65-915d-4acc-a7ef-ae4d93576cc8","rayCluster":"counter-auth-qmw8r","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:38:41.940Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"4c41ee65-915d-4acc-a7ef-ae4d93576cc8","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:38:38Z"},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","dashboardURL":"counter-auth-qmw8r-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:29Z","stateTransitionTimes":{"ready":"2026-11-03T15:38:29Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:38:12Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:38:29Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:38:38Z"},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","dashboardURL":"counter-auth-qmw8r-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:29Z","stateTransitionTimes":{"ready":"2026-11-03T15:38:29Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:38:12Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:38:29Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:38:44.941Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"d519e216-4ac4-4ec6-b16a-dea1f3e6ef29","JobStatus":"RUNNING","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:38:44.941Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"d519e216-4ac4-4ec6-b16a-dea1f3e6ef29","name":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:38:44.941Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"d519e216-4ac4-4ec6-b16a-dea1f3e6ef29","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:38:44.941Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"d519e216-4ac4-4ec6-b16a-dea1f3e6ef29"}
{"level":"info","ts":"2026-11-03T15:38:44.941Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"d519e216-4ac4-4ec6-b16a-dea1f3e6ef29","rayCluster":"counter-auth-qmw8r","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:38:44.944Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"d519e216-4ac4-4ec6-b16a-dea1f3e6ef29","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:38:38Z"},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","dashboardURL":"counter-auth-qmw8r-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"RUNNING","jobDeploymentStatus":"Running","message":"Job is currently running.","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:29Z","stateTransitionTimes":{"ready":"2026-11-03T15:38:29Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:38:12Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:38:29Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:38:38Z","endTime":"2026-11-03T15:38:42Z"},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","dashboardURL":"counter-auth-qmw8r-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:29Z","stateTransitionTimes":{"ready":"2026-11-03T15:38:29Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:38:12Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:38:29Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:38:44.944Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"d519e216-4ac4-4ec6-b16a-dea1f3e6ef29","old JobStatus":"RUNNING","new JobStatus":"SUCCEEDED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Running"}
{"level":"info","ts":"2026-11-03T15:38:45.041Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"46cffc44-8215-4609-99ce-ced0a0955b47","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:38:45.041Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"46cffc44-8215-4609-99ce-ced0a0955b47","name":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:38:45.041Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"46cffc44-8215-4609-99ce-ced0a0955b47","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:38:45.041Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"46cffc44-8215-4609-99ce-ced0a0955b47"}
{"level":"info","ts":"2026-11-03T15:38:45.041Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"46cffc44-8215-4609-99ce-ced0a0955b47","rayCluster":"counter-auth-qmw8r","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:38:45.044Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"46cffc44-8215-4609-99ce-ced0a0955b47","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:38:38Z","endTime":"2026-11-03T15:38:42Z"},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","dashboardURL":"counter-auth-qmw8r-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:29Z","stateTransitionTimes":{"ready":"2026-11-03T15:38:29Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:38:12Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:38:29Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:38:38Z","endTime":"2026-11-03T15:38:42Z"},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","dashboardURL":"counter-auth-qmw8r-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:29Z","stateTransitionTimes":{"ready":"2026-11-03T15:38:29Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:38:12Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:38:29Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:38:48.044Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"5818430a-9352-4a51-995f-d55f8621c032","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:38:48.044Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"5818430a-9352-4a51-995f-d55f8621c032","name":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:38:48.044Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"5818430a-9352-4a51-995f-d55f8621c032","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:38:48.045Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"5818430a-9352-4a51-995f-d55f8621c032"}
{"level":"info","ts":"2026-11-03T15:38:48.045Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"5818430a-9352-4a51-995f-d55f8621c032","rayCluster":"counter-auth-qmw8r","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:38:48.048Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"5818430a-9352-4a51-995f-d55f8621c032","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:38:38Z","endTime":"2026-11-03T15:38:42Z"},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","dashboardURL":"counter-auth-qmw8r-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:29Z","stateTransitionTimes":{"ready":"2026-11-03T15:38:29Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:38:12Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:38:29Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:38:38Z","endTime":"2026-11-03T15:38:42Z"},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","dashboardURL":"counter-auth-qmw8r-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:29Z","stateTransitionTimes":{"ready":"2026-11-03T15:38:29Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:38:12Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:38:29Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:38:48.752Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"78dc0505-5f58-4234-8e41-25431cd23637","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:38:48.753Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"78dc0505-5f58-4234-8e41-25431cd23637","name":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:38:48.753Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"78dc0505-5f58-4234-8e41-25431cd23637","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:38:48.753Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"78dc0505-5f58-4234-8e41-25431cd23637"}
{"level":"info","ts":"2026-11-03T15:38:48.754Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"78dc0505-5f58-4234-8e41-25431cd23637","rayCluster":"counter-auth-qmw8r","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:38:48.758Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"78dc0505-5f58-4234-8e41-25431cd23637","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:38:38Z","endTime":"2026-11-03T15:38:42Z"},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","dashboardURL":"counter-auth-qmw8r-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:29Z","stateTransitionTimes":{"ready":"2026-11-03T15:38:29Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:38:12Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:38:29Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:38:38Z","endTime":"2026-11-03T15:38:42Z"},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","dashboardURL":"counter-auth-qmw8r-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:29Z","stateTransitionTimes":{"ready":"2026-11-03T15:38:29Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:38:12Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:38:29Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:38:49.761Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"e488ca99-c10b-4717-9481-eefc00978a6f","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:38:49.761Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"e488ca99-c10b-4717-9481-eefc00978a6f","name":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:38:49.761Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"e488ca99-c10b-4717-9481-eefc00978a6f","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:38:49.761Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"e488ca99-c10b-4717-9481-eefc00978a6f"}
{"level":"info","ts":"2026-11-03T15:38:49.761Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"e488ca99-c10b-4717-9481-eefc00978a6f","rayCluster":"counter-auth-qmw8r","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:38:49.765Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"e488ca99-c10b-4717-9481-eefc00978a6f","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:38:38Z","endTime":"2026-11-03T15:38:42Z"},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","dashboardURL":"counter-auth-qmw8r-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:29Z","stateTransitionTimes":{"ready":"2026-11-03T15:38:29Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:38:12Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:38:29Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:38:38Z","endTime":"2026-11-03T15:38:42Z"},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","dashboardURL":"counter-auth-qmw8r-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:29Z","stateTransitionTimes":{"ready":"2026-11-03T15:38:29Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:38:12Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:38:29Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:38:49.794Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"d338529f-643c-4f52-9dd4-2ae74312342f","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Running","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:38:49.794Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"d338529f-643c-4f52-9dd4-2ae74312342f","name":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:38:49.794Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"d338529f-643c-4f52-9dd4-2ae74312342f","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:38:49.794Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"d338529f-643c-4f52-9dd4-2ae74312342f"}
{"level":"info","ts":"2026-11-03T15:38:49.794Z","logger":"controllers.RayJob","msg":"Service has already exists in the RayCluster, skip Update","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"d338529f-643c-4f52-9dd4-2ae74312342f","rayCluster":"counter-auth-qmw8r","serviceType":"headService"}
{"level":"info","ts":"2026-11-03T15:38:49.797Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"d338529f-643c-4f52-9dd4-2ae74312342f","oldRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:38:38Z","endTime":"2026-11-03T15:38:42Z"},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","dashboardURL":"counter-auth-qmw8r-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Running","message":"Job finished successfully.","startTime":"2026-11-03T15:36:30Z","succeeded":0,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:29Z","stateTransitionTimes":{"ready":"2026-11-03T15:38:29Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:38:12Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:38:29Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{"startTime":"2026-11-03T15:38:38Z","endTime":"2026-11-03T15:38:42Z"},"jobId":"counter-auth-lfm9q","rayClusterName":"counter-auth-qmw8r","dashboardURL":"counter-auth-qmw8r-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobStatus":"SUCCEEDED","jobDeploymentStatus":"Complete","message":"Job finished successfully.","startTime":"2026-11-03T15:36:30Z","succeeded":1,"failed":0,"rayClusterStatus":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:29Z","stateTransitionTimes":{"ready":"2026-11-03T15:38:29Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.53","serviceIP":"10.134.0.53","podName":"counter-auth-qmw8r-head-b964f","serviceName":"counter-auth-qmw8r-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:38:12Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:38:29Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:36:30Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:36:31Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:38:49.797Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"d338529f-643c-4f52-9dd4-2ae74312342f","old JobStatus":"SUCCEEDED","new JobStatus":"SUCCEEDED","old JobDeploymentStatus":"Running","new JobDeploymentStatus":"Complete"}
{"level":"info","ts":"2026-11-03T15:38:49.861Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"4e2c4404-77ae-4d3a-8d1d-e5ac71be8de8","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:38:49.861Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"4e2c4404-77ae-4d3a-8d1d-e5ac71be8de8","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Complete","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-11-03 15:38:49 +0000 UTC","Now":"2026-11-03T15:38:49.861Z","ShutdownTime":"2026-11-03T15:38:49.000Z"}
{"level":"info","ts":"2026-11-03T15:38:49.921Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob is deleted","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"4e2c4404-77ae-4d3a-8d1d-e5ac71be8de8","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:38:49.921Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"4e2c4404-77ae-4d3a-8d1d-e5ac71be8de8","isClusterDeleted":false}
{"level":"info","ts":"2026-11-03T15:38:49.921Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"4e2c4404-77ae-4d3a-8d1d-e5ac71be8de8","deletionMechanism":"ShutdownAfterJobFinishes","RayCluster":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:38:49.922Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"reconcileID":"25081dbc-e182-4eac-8b9a-1cc34ce28196"}
{"level":"info","ts":"2026-11-03T15:38:49.922Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"counter-auth-qmw8r","reconcileID":"ea3feab8-46ff-4a5e-a810-ed3172acac8e","namespacedName":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:38:49.922Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"counter-auth-qmw8r","reconcileID":"ea3feab8-46ff-4a5e-a810-ed3172acac8e","cluster":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:38:49.922Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"counter-auth-qmw8r","reconcileID":"ea3feab8-46ff-4a5e-a810-ed3172acac8e","cluster":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:38:49.922Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"446b7cb6-002e-44d1-ad74-5b9e60b5ecd9","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:38:49.922Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"446b7cb6-002e-44d1-ad74-5b9e60b5ecd9","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Complete","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-11-03 15:38:49 +0000 UTC","Now":"2026-11-03T15:38:49.922Z","ShutdownTime":"2026-11-03T15:38:49.000Z"}
{"level":"info","ts":"2026-11-03T15:38:49.922Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"counter-auth-qmw8r","reconcileID":"ea3feab8-46ff-4a5e-a810-ed3172acac8e","namespace":"test-ns-kbx4w","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-11-03T15:38:49.922Z","logger":"controllers.RayJob","msg":"The deletion of the associated RayCluster for RayJob is ongoing.","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"446b7cb6-002e-44d1-ad74-5b9e60b5ecd9","RayCluster":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:38:49.922Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"446b7cb6-002e-44d1-ad74-5b9e60b5ecd9","isClusterDeleted":false}
{"level":"info","ts":"2026-11-03T15:38:49.922Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"446b7cb6-002e-44d1-ad74-5b9e60b5ecd9","deletionMechanism":"ShutdownAfterJobFinishes","RayCluster":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:38:49.975Z","logger":"authentication-controller","msg":"Deleted shared ReferenceGrant (last cluster with auth)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"counter-auth-qmw8r","reconcileID":"ea3feab8-46ff-4a5e-a810-ed3172acac8e","name":"kuberay-gateway-access","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:38:50.035Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"counter-auth-qmw8r","reconcileID":"ea3feab8-46ff-4a5e-a810-ed3172acac8e","name":"test-ns-kbx4w-counter-auth-qmw8r","namespace":"opendatahub"}
{"level":"info","ts":"2026-11-03T15:38:50.087Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"counter-auth-qmw8r","reconcileID":"ea3feab8-46ff-4a5e-a810-ed3172acac8e","configMap":"kube-rbac-proxy-config-counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:38:50.139Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"counter-auth-qmw8r","reconcileID":"ea3feab8-46ff-4a5e-a810-ed3172acac8e","serviceAccount":"counter-auth-qmw8r-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-11-03T15:38:50.198Z","logger":"raycluster-resource","msg":"default","name":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:38:50.198Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:38:50.255Z","logger":"raycluster-resource","msg":"validate update","name":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:38:50.310Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"counter-auth-qmw8r","reconcileID":"ea3feab8-46ff-4a5e-a810-ed3172acac8e","cluster":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:38:50.310Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"counter-auth-qmw8r","reconcileID":"030cd578-a183-4d5f-98e5-b593932256ab","namespacedName":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:38:50.310Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"counter-auth-qmw8r","reconcileID":"030cd578-a183-4d5f-98e5-b593932256ab","cluster":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:38:50.310Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"counter-auth-qmw8r","reconcileID":"030cd578-a183-4d5f-98e5-b593932256ab","cluster":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:38:50.310Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"counter-auth-qmw8r","reconcileID":"030cd578-a183-4d5f-98e5-b593932256ab","namespace":"test-ns-kbx4w","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-11-03T15:38:50.310Z","logger":"authentication-controller","msg":"ReferenceGrant already deleted","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"counter-auth-qmw8r","reconcileID":"030cd578-a183-4d5f-98e5-b593932256ab","name":"kuberay-gateway-access"}
{"level":"info","ts":"2026-11-03T15:38:50.311Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"9b491c6c-7122-4965-8381-668b6cce93fa","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:38:50.311Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"9b491c6c-7122-4965-8381-668b6cce93fa","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Complete","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-11-03 15:38:49 +0000 UTC","Now":"2026-11-03T15:38:50.311Z","ShutdownTime":"2026-11-03T15:38:49.000Z"}
{"level":"info","ts":"2026-11-03T15:38:50.311Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"9b491c6c-7122-4965-8381-668b6cce93fa","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:38:50.311Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"9b491c6c-7122-4965-8381-668b6cce93fa","isClusterDeleted":true}
{"level":"info","ts":"2026-11-03T15:38:50.311Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"9b491c6c-7122-4965-8381-668b6cce93fa","deletionMechanism":"ShutdownAfterJobFinishes","RayCluster":"counter-auth-qmw8r"}
{"level":"error","ts":"2026-11-03T15:38:50.361Z","logger":"authentication-controller","msg":"Failed to remove finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"counter-auth-qmw8r","reconcileID":"030cd578-a183-4d5f-98e5-b593932256ab","error":"rayclusters.ray.io \"counter-auth-qmw8r\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleDeletion\n\t/workspace/controllers/ray/authentication_controller.go:195\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:38:50.361Z","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"counter-auth-qmw8r","reconcileID":"030cd578-a183-4d5f-98e5-b593932256ab"}
{"level":"error","ts":"2026-11-03T15:38:50.361Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"counter-auth-qmw8r","reconcileID":"030cd578-a183-4d5f-98e5-b593932256ab","error":"rayclusters.ray.io \"counter-auth-qmw8r\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:38:50.361Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"counter-auth-qmw8r","reconcileID":"b3ed472a-7ef6-430e-b005-61964513118b","namespacedName":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:38:50.361Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"counter-auth-qmw8r","reconcileID":"b3ed472a-7ef6-430e-b005-61964513118b"}
{"level":"info","ts":"2026-11-03T15:38:50.361Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"counter-auth-qmw8r","reconcileID":"b3ed472a-7ef6-430e-b005-61964513118b","namespace":"test-ns-kbx4w","authClusters":0}
{"level":"info","ts":"2026-11-03T15:38:50.361Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"counter-auth-qmw8r","reconcileID":"b3ed472a-7ef6-430e-b005-61964513118b","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:38:50.859Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"a56f5496-9122-44b9-b03e-a06dd146cbc0","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:38:50.859Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"a56f5496-9122-44b9-b03e-a06dd146cbc0","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-11-03T15:38:50.916Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"a56f5496-9122-44b9-b03e-a06dd146cbc0"}
{"level":"info","ts":"2026-11-03T15:38:50.916Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"a56f5496-9122-44b9-b03e-a06dd146cbc0","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-11-03T15:38:50.916Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"a56f5496-9122-44b9-b03e-a06dd146cbc0","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"ttl-initializing-jj9hd","rayClusterName":"ttl-initializing-vl6x2","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:50Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:38:50.916Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"a56f5496-9122-44b9-b03e-a06dd146cbc0","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:38:50.975Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"5bbe68f2-82ed-4cc6-b5fe-52fcb9c32a21","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:38:50.975Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"5bbe68f2-82ed-4cc6-b5fe-52fcb9c32a21","name":"ttl-initializing-vl6x2"}
{"level":"info","ts":"2026-11-03T15:38:50.975Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"5bbe68f2-82ed-4cc6-b5fe-52fcb9c32a21","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:38:50.975Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"5bbe68f2-82ed-4cc6-b5fe-52fcb9c32a21","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:38:51.031Z","logger":"raycluster-resource","msg":"default","name":"ttl-initializing-vl6x2"}
{"level":"info","ts":"2026-11-03T15:38:51.031Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:38:51.087Z","logger":"raycluster-resource","msg":"validate create","name":"ttl-initializing-vl6x2"}
{"level":"info","ts":"2026-11-03T15:38:51.141Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"5bbe68f2-82ed-4cc6-b5fe-52fcb9c32a21","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:38:51.141Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"5f439ff7-80ee-4ba1-8268-3e0b11b8e2ed"}
{"level":"info","ts":"2026-11-03T15:38:51.141Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"5f439ff7-80ee-4ba1-8268-3e0b11b8e2ed"}
{"level":"info","ts":"2026-11-03T15:38:51.141Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"14bdc479-4bba-4dbc-a81d-4290c0a9e238","cluster":"ttl-initializing-vl6x2"}
{"level":"info","ts":"2026-11-03T15:38:51.141Z","logger":"controllers.NetworkPolicy","msg":"Reconciling NetworkPolicies for RayCluster","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"89dc80dc-ac40-4a2b-88f2-e8c9bb5719e0","cluster":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:38:51.141Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"5bbe68f2-82ed-4cc6-b5fe-52fcb9c32a21"}
{"level":"info","ts":"2026-11-03T15:38:51.142Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"5bbe68f2-82ed-4cc6-b5fe-52fcb9c32a21","RayCluster":"ttl-initializing-vl6x2","State":""}
{"level":"info","ts":"2026-11-03T15:38:51.142Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"5bbe68f2-82ed-4cc6-b5fe-52fcb9c32a21","oldRayJobStatus":{"rayJobInfo":{},"jobId":"ttl-initializing-jj9hd","rayClusterName":"ttl-initializing-vl6x2","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:50Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"ttl-initializing-jj9hd","rayClusterName":"ttl-initializing-vl6x2","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:50Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:38:51.142Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"ae58ff92-21fd-4938-81d4-b2089a9d3ee5","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:38:51.142Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"ae58ff92-21fd-4938-81d4-b2089a9d3ee5","name":"ttl-initializing-vl6x2"}
{"level":"info","ts":"2026-11-03T15:38:51.142Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-initializing-vl6x2","reconcileID":"5b909c95-24e7-4904-85bb-83ef7ec0d443","namespacedName":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:38:51.142Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"ae58ff92-21fd-4938-81d4-b2089a9d3ee5","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:38:51.142Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-initializing-vl6x2","reconcileID":"5b909c95-24e7-4904-85bb-83ef7ec0d443","mode":"IntegratedOAuth","cluster":"ttl-initializing-vl6x2"}
{"level":"info","ts":"2026-11-03T15:38:51.142Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-initializing-vl6x2","reconcileID":"5b909c95-24e7-4904-85bb-83ef7ec0d443","cluster":"ttl-initializing-vl6x2"}
{"level":"info","ts":"2026-11-03T15:38:51.142Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-initializing-vl6x2","reconcileID":"5b909c95-24e7-4904-85bb-83ef7ec0d443","cluster":"ttl-initializing-vl6x2"}
{"level":"info","ts":"2026-11-03T15:38:51.142Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"ae58ff92-21fd-4938-81d4-b2089a9d3ee5"}
{"level":"info","ts":"2026-11-03T15:38:51.142Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"ae58ff92-21fd-4938-81d4-b2089a9d3ee5","RayCluster":"ttl-initializing-vl6x2","State":""}
{"level":"info","ts":"2026-11-03T15:38:51.142Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"ae58ff92-21fd-4938-81d4-b2089a9d3ee5","oldRayJobStatus":{"rayJobInfo":{},"jobId":"ttl-initializing-jj9hd","rayClusterName":"ttl-initializing-vl6x2","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:50Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"ttl-initializing-jj9hd","rayClusterName":"ttl-initializing-vl6x2","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:50Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:38:51.142Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"9913b730-0be3-4a9a-a92b-306104c56e45","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:38:51.142Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"9913b730-0be3-4a9a-a92b-306104c56e45","name":"ttl-initializing-vl6x2"}
{"level":"info","ts":"2026-11-03T15:38:51.142Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"9913b730-0be3-4a9a-a92b-306104c56e45","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:38:51.142Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"9913b730-0be3-4a9a-a92b-306104c56e45"}
{"level":"info","ts":"2026-11-03T15:38:51.142Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"9913b730-0be3-4a9a-a92b-306104c56e45","RayCluster":"ttl-initializing-vl6x2","State":""}
{"level":"info","ts":"2026-11-03T15:38:51.142Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"9913b730-0be3-4a9a-a92b-306104c56e45","oldRayJobStatus":{"rayJobInfo":{},"jobId":"ttl-initializing-jj9hd","rayClusterName":"ttl-initializing-vl6x2","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:50Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"ttl-initializing-jj9hd","rayClusterName":"ttl-initializing-vl6x2","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:50Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:38:51.197Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"5f439ff7-80ee-4ba1-8268-3e0b11b8e2ed","name":"ttl-initializing-vl6x2-head-svc"}
{"level":"info","ts":"2026-11-03T15:38:51.198Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"5f439ff7-80ee-4ba1-8268-3e0b11b8e2ed"}
{"level":"info","ts":"2026-11-03T15:38:51.198Z","logger":"controllers.NetworkPolicy","msg":"Successfully created NetworkPolicy","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"89dc80dc-ac40-4a2b-88f2-e8c9bb5719e0","name":"ttl-initializing-vl6x2-head"}
{"level":"info","ts":"2026-11-03T15:38:51.199Z","logger":"raycluster-resource","msg":"default","name":"ttl-initializing-vl6x2"}
{"level":"info","ts":"2026-11-03T15:38:51.199Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:38:51.247Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"5f439ff7-80ee-4ba1-8268-3e0b11b8e2ed","error":"certificate ray-head-cert-ttl-initializing-vl6x2 not found: certificates.cert-manager.io \"ray-head-cert-ttl-initializing-vl6x2\" not found"}
{"level":"error","ts":"2026-11-03T15:38:51.247Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"5f439ff7-80ee-4ba1-8268-3e0b11b8e2ed","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-ttl-initializing-vl6x2 not found: certificates.cert-manager.io \"ray-head-cert-ttl-initializing-vl6x2\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:38:51.247Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"5f439ff7-80ee-4ba1-8268-3e0b11b8e2ed","error":"mTLS secrets not ready: certificate ray-head-cert-ttl-initializing-vl6x2 not found: certificates.cert-manager.io \"ray-head-cert-ttl-initializing-vl6x2\" not found"}
{"level":"info","ts":"2026-11-03T15:38:51.247Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"5f439ff7-80ee-4ba1-8268-3e0b11b8e2ed","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"ttl-initializing-vl6x2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:38:51.247Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"5f439ff7-80ee-4ba1-8268-3e0b11b8e2ed","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"ttl-initializing-vl6x2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:38:51.247Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"5f439ff7-80ee-4ba1-8268-3e0b11b8e2ed","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"ttl-initializing-vl6x2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:38:51.247Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"5f439ff7-80ee-4ba1-8268-3e0b11b8e2ed","name":"ttl-initializing-vl6x2","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:51Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ttl-initializing-vl6x2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:38:51.255Z","logger":"controllers.NetworkPolicy","msg":"Successfully created NetworkPolicy","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"89dc80dc-ac40-4a2b-88f2-e8c9bb5719e0","name":"ttl-initializing-vl6x2-workers-small-group"}
{"level":"info","ts":"2026-11-03T15:38:51.255Z","logger":"controllers.NetworkPolicy","msg":"Successfully reconciled NetworkPolicies for RayCluster","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"89dc80dc-ac40-4a2b-88f2-e8c9bb5719e0","cluster":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:38:51.255Z","logger":"controllers.NetworkPolicy","msg":"Reconciling NetworkPolicies for RayCluster","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"89b057f8-87cd-480c-847a-77a78b4d2adb","cluster":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:38:51.263Z","logger":"raycluster-resource","msg":"validate update","name":"ttl-initializing-vl6x2"}
{"level":"info","ts":"2026-11-03T15:38:51.307Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"5f439ff7-80ee-4ba1-8268-3e0b11b8e2ed"}
{"level":"error","ts":"2026-11-03T15:38:51.307Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"5f439ff7-80ee-4ba1-8268-3e0b11b8e2ed","error":"mTLS secrets not ready: certificate ray-head-cert-ttl-initializing-vl6x2 not found: certificates.cert-manager.io \"ray-head-cert-ttl-initializing-vl6x2\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:38:51.307Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"1c0492db-1552-47a8-84b1-7b5601dbe7d0"}
{"level":"info","ts":"2026-11-03T15:38:51.307Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"1c0492db-1552-47a8-84b1-7b5601dbe7d0"}
{"level":"info","ts":"2026-11-03T15:38:51.307Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"1c0492db-1552-47a8-84b1-7b5601dbe7d0","1 head service found":"ttl-initializing-vl6x2-head-svc"}
{"level":"info","ts":"2026-11-03T15:38:51.307Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"1c0492db-1552-47a8-84b1-7b5601dbe7d0"}
{"level":"info","ts":"2026-11-03T15:38:51.307Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"b67cc1f2-ea46-486b-8597-e960996ce906","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:38:51.307Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"b67cc1f2-ea46-486b-8597-e960996ce906","name":"ttl-initializing-vl6x2"}
{"level":"info","ts":"2026-11-03T15:38:51.307Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"b67cc1f2-ea46-486b-8597-e960996ce906","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:38:51.307Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"b67cc1f2-ea46-486b-8597-e960996ce906"}
{"level":"info","ts":"2026-11-03T15:38:51.307Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"b67cc1f2-ea46-486b-8597-e960996ce906","RayCluster":"ttl-initializing-vl6x2","State":""}
{"level":"info","ts":"2026-11-03T15:38:51.307Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"b67cc1f2-ea46-486b-8597-e960996ce906","oldRayJobStatus":{"rayJobInfo":{},"jobId":"ttl-initializing-jj9hd","rayClusterName":"ttl-initializing-vl6x2","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:50Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"ttl-initializing-jj9hd","rayClusterName":"ttl-initializing-vl6x2","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:50Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:51Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ttl-initializing-vl6x2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:38:51.307Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"b67cc1f2-ea46-486b-8597-e960996ce906","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:38:51.324Z","logger":"raycluster-resource","msg":"default","name":"ttl-initializing-vl6x2"}
{"level":"info","ts":"2026-11-03T15:38:51.324Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:38:51.357Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"1c0492db-1552-47a8-84b1-7b5601dbe7d0","error":"certificate ray-head-cert-ttl-initializing-vl6x2 not found: certificates.cert-manager.io \"ray-head-cert-ttl-initializing-vl6x2\" not found"}
{"level":"error","ts":"2026-11-03T15:38:51.357Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"1c0492db-1552-47a8-84b1-7b5601dbe7d0","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-ttl-initializing-vl6x2 not found: certificates.cert-manager.io \"ray-head-cert-ttl-initializing-vl6x2\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:38:51.357Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"1c0492db-1552-47a8-84b1-7b5601dbe7d0","error":"mTLS secrets not ready: certificate ray-head-cert-ttl-initializing-vl6x2 not found: certificates.cert-manager.io \"ray-head-cert-ttl-initializing-vl6x2\" not found"}
{"level":"info","ts":"2026-11-03T15:38:51.357Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"1c0492db-1552-47a8-84b1-7b5601dbe7d0","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"ttl-initializing-vl6x2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:38:51.357Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"1c0492db-1552-47a8-84b1-7b5601dbe7d0","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"ttl-initializing-vl6x2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:38:51.357Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"1c0492db-1552-47a8-84b1-7b5601dbe7d0","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"ttl-initializing-vl6x2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:38:51.357Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"1c0492db-1552-47a8-84b1-7b5601dbe7d0"}
{"level":"error","ts":"2026-11-03T15:38:51.357Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"1c0492db-1552-47a8-84b1-7b5601dbe7d0","error":"mTLS secrets not ready: certificate ray-head-cert-ttl-initializing-vl6x2 not found: certificates.cert-manager.io \"ray-head-cert-ttl-initializing-vl6x2\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:38:51.370Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"cdfe4184-92ed-458e-a62a-98a18f8e5f7c"}
{"level":"info","ts":"2026-11-03T15:38:51.371Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"cdfe4184-92ed-458e-a62a-98a18f8e5f7c"}
{"level":"info","ts":"2026-11-03T15:38:51.371Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"cdfe4184-92ed-458e-a62a-98a18f8e5f7c","1 head service found":"ttl-initializing-vl6x2-head-svc"}
{"level":"info","ts":"2026-11-03T15:38:51.371Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"cdfe4184-92ed-458e-a62a-98a18f8e5f7c"}
{"level":"info","ts":"2026-11-03T15:38:51.374Z","logger":"controllers.NetworkPolicy","msg":"Successfully reconciled NetworkPolicies for RayCluster","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"89b057f8-87cd-480c-847a-77a78b4d2adb","cluster":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"}
{"level":"error","ts":"2026-11-03T15:38:51.375Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-initializing-vl6x2","reconcileID":"5b909c95-24e7-4904-85bb-83ef7ec0d443","error":"failed to add finalizer: Operation cannot be fulfilled on rayclusters.ray.io \"ttl-initializing-vl6x2\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:167\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:38:51.375Z","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-initializing-vl6x2","reconcileID":"5b909c95-24e7-4904-85bb-83ef7ec0d443"}
{"level":"error","ts":"2026-11-03T15:38:51.375Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-initializing-vl6x2","reconcileID":"5b909c95-24e7-4904-85bb-83ef7ec0d443","error":"failed to add finalizer: Operation cannot be fulfilled on rayclusters.ray.io \"ttl-initializing-vl6x2\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:38:51.375Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-initializing-vl6x2","reconcileID":"a749cc8a-0d66-4ed7-9894-923f80bb9cb1","namespacedName":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:38:51.375Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-initializing-vl6x2","reconcileID":"a749cc8a-0d66-4ed7-9894-923f80bb9cb1","mode":"IntegratedOAuth","cluster":"ttl-initializing-vl6x2"}
{"level":"info","ts":"2026-11-03T15:38:51.375Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-initializing-vl6x2","reconcileID":"a749cc8a-0d66-4ed7-9894-923f80bb9cb1","cluster":"ttl-initializing-vl6x2"}
{"level":"info","ts":"2026-11-03T15:38:51.375Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-initializing-vl6x2","reconcileID":"a749cc8a-0d66-4ed7-9894-923f80bb9cb1","cluster":"ttl-initializing-vl6x2"}
{"level":"info","ts":"2026-11-03T15:38:51.375Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"14304f1e-1651-40d8-961d-2d03086a4566","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:38:51.375Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"14304f1e-1651-40d8-961d-2d03086a4566","name":"ttl-initializing-vl6x2"}
{"level":"info","ts":"2026-11-03T15:38:51.375Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"14304f1e-1651-40d8-961d-2d03086a4566","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:38:51.376Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"14304f1e-1651-40d8-961d-2d03086a4566"}
{"level":"info","ts":"2026-11-03T15:38:51.376Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"14304f1e-1651-40d8-961d-2d03086a4566","RayCluster":"ttl-initializing-vl6x2","State":""}
{"level":"info","ts":"2026-11-03T15:38:51.376Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"14304f1e-1651-40d8-961d-2d03086a4566","oldRayJobStatus":{"rayJobInfo":{},"jobId":"ttl-initializing-jj9hd","rayClusterName":"ttl-initializing-vl6x2","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:50Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:51Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ttl-initializing-vl6x2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"ttl-initializing-jj9hd","rayClusterName":"ttl-initializing-vl6x2","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:50Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:51Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ttl-initializing-vl6x2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:38:51.423Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"cdfe4184-92ed-458e-a62a-98a18f8e5f7c","error":"certificate ray-head-cert-ttl-initializing-vl6x2 not found: certificates.cert-manager.io \"ray-head-cert-ttl-initializing-vl6x2\" not found"}
{"level":"error","ts":"2026-11-03T15:38:51.424Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"cdfe4184-92ed-458e-a62a-98a18f8e5f7c","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-ttl-initializing-vl6x2 not found: certificates.cert-manager.io \"ray-head-cert-ttl-initializing-vl6x2\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:38:51.424Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"cdfe4184-92ed-458e-a62a-98a18f8e5f7c","error":"mTLS secrets not ready: certificate ray-head-cert-ttl-initializing-vl6x2 not found: certificates.cert-manager.io \"ray-head-cert-ttl-initializing-vl6x2\" not found"}
{"level":"info","ts":"2026-11-03T15:38:51.424Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"cdfe4184-92ed-458e-a62a-98a18f8e5f7c","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"ttl-initializing-vl6x2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:38:51.424Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"cdfe4184-92ed-458e-a62a-98a18f8e5f7c","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"ttl-initializing-vl6x2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:38:51.424Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"cdfe4184-92ed-458e-a62a-98a18f8e5f7c","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"ttl-initializing-vl6x2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:38:51.424Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"cdfe4184-92ed-458e-a62a-98a18f8e5f7c"}
{"level":"error","ts":"2026-11-03T15:38:51.424Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"cdfe4184-92ed-458e-a62a-98a18f8e5f7c","error":"mTLS secrets not ready: certificate ray-head-cert-ttl-initializing-vl6x2 not found: certificates.cert-manager.io \"ray-head-cert-ttl-initializing-vl6x2\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:38:51.434Z","logger":"raycluster-resource","msg":"default","name":"ttl-initializing-vl6x2"}
{"level":"info","ts":"2026-11-03T15:38:51.434Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:38:51.444Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"90196d17-c419-40fa-8e7e-105d6f058c14"}
{"level":"info","ts":"2026-11-03T15:38:51.444Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"90196d17-c419-40fa-8e7e-105d6f058c14"}
{"level":"info","ts":"2026-11-03T15:38:51.444Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"90196d17-c419-40fa-8e7e-105d6f058c14","1 head service found":"ttl-initializing-vl6x2-head-svc"}
{"level":"info","ts":"2026-11-03T15:38:51.444Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"90196d17-c419-40fa-8e7e-105d6f058c14"}
{"level":"info","ts":"2026-11-03T15:38:51.490Z","logger":"raycluster-resource","msg":"validate update","name":"ttl-initializing-vl6x2"}
{"level":"info","ts":"2026-11-03T15:38:51.494Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"90196d17-c419-40fa-8e7e-105d6f058c14","error":"certificate ray-head-cert-ttl-initializing-vl6x2 not found: certificates.cert-manager.io \"ray-head-cert-ttl-initializing-vl6x2\" not found"}
{"level":"error","ts":"2026-11-03T15:38:51.494Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"90196d17-c419-40fa-8e7e-105d6f058c14","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-ttl-initializing-vl6x2 not found: certificates.cert-manager.io \"ray-head-cert-ttl-initializing-vl6x2\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:38:51.494Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"90196d17-c419-40fa-8e7e-105d6f058c14","error":"mTLS secrets not ready: certificate ray-head-cert-ttl-initializing-vl6x2 not found: certificates.cert-manager.io \"ray-head-cert-ttl-initializing-vl6x2\" not found"}
{"level":"info","ts":"2026-11-03T15:38:51.494Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"90196d17-c419-40fa-8e7e-105d6f058c14","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"ttl-initializing-vl6x2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:38:51.494Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"90196d17-c419-40fa-8e7e-105d6f058c14","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"ttl-initializing-vl6x2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:38:51.494Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"90196d17-c419-40fa-8e7e-105d6f058c14","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"ttl-initializing-vl6x2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:38:51.494Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"90196d17-c419-40fa-8e7e-105d6f058c14"}
{"level":"error","ts":"2026-11-03T15:38:51.494Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"90196d17-c419-40fa-8e7e-105d6f058c14","error":"mTLS secrets not ready: certificate ray-head-cert-ttl-initializing-vl6x2 not found: certificates.cert-manager.io \"ray-head-cert-ttl-initializing-vl6x2\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:38:51.535Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"51d91d91-5210-41bb-9dc8-2d6148593e65"}
{"level":"info","ts":"2026-11-03T15:38:51.535Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"51d91d91-5210-41bb-9dc8-2d6148593e65"}
{"level":"info","ts":"2026-11-03T15:38:51.535Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"51d91d91-5210-41bb-9dc8-2d6148593e65","1 head service found":"ttl-initializing-vl6x2-head-svc"}
{"level":"info","ts":"2026-11-03T15:38:51.535Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"51d91d91-5210-41bb-9dc8-2d6148593e65"}
{"level":"info","ts":"2026-11-03T15:38:51.545Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"6a5014a3-732f-4ae8-8988-9f50a1d577a8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:38:51.545Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"6a5014a3-732f-4ae8-8988-9f50a1d577a8","name":"ttl-initializing-vl6x2"}
{"level":"info","ts":"2026-11-03T15:38:51.545Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"6a5014a3-732f-4ae8-8988-9f50a1d577a8","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:38:51.545Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"6a5014a3-732f-4ae8-8988-9f50a1d577a8"}
{"level":"info","ts":"2026-11-03T15:38:51.545Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"6a5014a3-732f-4ae8-8988-9f50a1d577a8","RayCluster":"ttl-initializing-vl6x2","State":""}
{"level":"info","ts":"2026-11-03T15:38:51.545Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"6a5014a3-732f-4ae8-8988-9f50a1d577a8","oldRayJobStatus":{"rayJobInfo":{},"jobId":"ttl-initializing-jj9hd","rayClusterName":"ttl-initializing-vl6x2","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:50Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:51Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ttl-initializing-vl6x2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"ttl-initializing-jj9hd","rayClusterName":"ttl-initializing-vl6x2","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:50Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:51Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ttl-initializing-vl6x2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:38:51.588Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"51d91d91-5210-41bb-9dc8-2d6148593e65","error":"certificate ray-head-cert-ttl-initializing-vl6x2 not found: certificates.cert-manager.io \"ray-head-cert-ttl-initializing-vl6x2\" not found"}
{"level":"error","ts":"2026-11-03T15:38:51.588Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"51d91d91-5210-41bb-9dc8-2d6148593e65","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-ttl-initializing-vl6x2 not found: certificates.cert-manager.io \"ray-head-cert-ttl-initializing-vl6x2\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:38:51.588Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"51d91d91-5210-41bb-9dc8-2d6148593e65","error":"mTLS secrets not ready: certificate ray-head-cert-ttl-initializing-vl6x2 not found: certificates.cert-manager.io \"ray-head-cert-ttl-initializing-vl6x2\" not found"}
{"level":"info","ts":"2026-11-03T15:38:51.588Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"51d91d91-5210-41bb-9dc8-2d6148593e65","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"ttl-initializing-vl6x2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:38:51.588Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"51d91d91-5210-41bb-9dc8-2d6148593e65","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"ttl-initializing-vl6x2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:38:51.588Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"51d91d91-5210-41bb-9dc8-2d6148593e65","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"ttl-initializing-vl6x2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:38:51.588Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"51d91d91-5210-41bb-9dc8-2d6148593e65"}
{"level":"error","ts":"2026-11-03T15:38:51.588Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"51d91d91-5210-41bb-9dc8-2d6148593e65","error":"mTLS secrets not ready: certificate ray-head-cert-ttl-initializing-vl6x2 not found: certificates.cert-manager.io \"ray-head-cert-ttl-initializing-vl6x2\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:38:51.596Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-initializing-vl6x2","reconcileID":"a749cc8a-0d66-4ed7-9894-923f80bb9cb1","name":"ttl-initializing-vl6x2-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-11-03T15:38:51.647Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-initializing-vl6x2","reconcileID":"a749cc8a-0d66-4ed7-9894-923f80bb9cb1","name":"kuberay-gateway-access","namespace":"test-ns-kbx4w","operation":"created"}
{"level":"info","ts":"2026-11-03T15:38:51.669Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"19d20a4a-6924-4778-bd1b-4a9c0cd61eb9"}
{"level":"info","ts":"2026-11-03T15:38:51.669Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"19d20a4a-6924-4778-bd1b-4a9c0cd61eb9"}
{"level":"info","ts":"2026-11-03T15:38:51.669Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"19d20a4a-6924-4778-bd1b-4a9c0cd61eb9","1 head service found":"ttl-initializing-vl6x2-head-svc"}
{"level":"info","ts":"2026-11-03T15:38:51.669Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"19d20a4a-6924-4778-bd1b-4a9c0cd61eb9"}
{"level":"info","ts":"2026-11-03T15:38:51.708Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-initializing-vl6x2","reconcileID":"a749cc8a-0d66-4ed7-9894-923f80bb9cb1","name":"test-ns-kbx4w-ttl-initializing-vl6x2","operation":"created"}
{"level":"info","ts":"2026-11-03T15:38:51.719Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"19d20a4a-6924-4778-bd1b-4a9c0cd61eb9","error":"certificate ray-head-cert-ttl-initializing-vl6x2 not found: certificates.cert-manager.io \"ray-head-cert-ttl-initializing-vl6x2\" not found"}
{"level":"error","ts":"2026-11-03T15:38:51.719Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"19d20a4a-6924-4778-bd1b-4a9c0cd61eb9","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-ttl-initializing-vl6x2 not found: certificates.cert-manager.io \"ray-head-cert-ttl-initializing-vl6x2\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:38:51.719Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"19d20a4a-6924-4778-bd1b-4a9c0cd61eb9","error":"mTLS secrets not ready: certificate ray-head-cert-ttl-initializing-vl6x2 not found: certificates.cert-manager.io \"ray-head-cert-ttl-initializing-vl6x2\" not found"}
{"level":"info","ts":"2026-11-03T15:38:51.719Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"19d20a4a-6924-4778-bd1b-4a9c0cd61eb9","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"ttl-initializing-vl6x2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:38:51.719Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"19d20a4a-6924-4778-bd1b-4a9c0cd61eb9","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"ttl-initializing-vl6x2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:38:51.719Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"19d20a4a-6924-4778-bd1b-4a9c0cd61eb9","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"ttl-initializing-vl6x2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:38:51.719Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"19d20a4a-6924-4778-bd1b-4a9c0cd61eb9"}
{"level":"error","ts":"2026-11-03T15:38:51.719Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"19d20a4a-6924-4778-bd1b-4a9c0cd61eb9","error":"mTLS secrets not ready: certificate ray-head-cert-ttl-initializing-vl6x2 not found: certificates.cert-manager.io \"ray-head-cert-ttl-initializing-vl6x2\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:38:51.760Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-initializing-vl6x2","reconcileID":"a749cc8a-0d66-4ed7-9894-923f80bb9cb1","name":"kube-rbac-proxy-config-ttl-initializing-vl6x2","operation":"created"}
{"level":"error","ts":"2026-11-03T15:38:51.816Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-initializing-vl6x2","reconcileID":"a749cc8a-0d66-4ed7-9894-923f80bb9cb1","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"ttl-initializing-vl6x2\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:167\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:38:51.816Z","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-initializing-vl6x2","reconcileID":"a749cc8a-0d66-4ed7-9894-923f80bb9cb1"}
{"level":"error","ts":"2026-11-03T15:38:51.816Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-initializing-vl6x2","reconcileID":"a749cc8a-0d66-4ed7-9894-923f80bb9cb1","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"ttl-initializing-vl6x2\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:38:51.816Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-initializing-vl6x2","reconcileID":"a41a07f0-074f-4d84-913b-75f976a20462","namespacedName":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:38:51.816Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-initializing-vl6x2","reconcileID":"a41a07f0-074f-4d84-913b-75f976a20462","mode":"IntegratedOAuth","cluster":"ttl-initializing-vl6x2"}
{"level":"info","ts":"2026-11-03T15:38:51.816Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-initializing-vl6x2","reconcileID":"a41a07f0-074f-4d84-913b-75f976a20462","cluster":"ttl-initializing-vl6x2"}
{"level":"info","ts":"2026-11-03T15:38:51.879Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"01ef4835-c9d8-44bf-b9d1-2fbbaf2b8d7d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:38:51.879Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"01ef4835-c9d8-44bf-b9d1-2fbbaf2b8d7d","name":"ttl-initializing-vl6x2"}
{"level":"info","ts":"2026-11-03T15:38:51.879Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"01ef4835-c9d8-44bf-b9d1-2fbbaf2b8d7d","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:38:51.879Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"01ef4835-c9d8-44bf-b9d1-2fbbaf2b8d7d"}
{"level":"info","ts":"2026-11-03T15:38:51.879Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"01ef4835-c9d8-44bf-b9d1-2fbbaf2b8d7d","RayCluster":"ttl-initializing-vl6x2","State":""}
{"level":"info","ts":"2026-11-03T15:38:51.879Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"01ef4835-c9d8-44bf-b9d1-2fbbaf2b8d7d","oldRayJobStatus":{"rayJobInfo":{},"jobId":"ttl-initializing-jj9hd","rayClusterName":"ttl-initializing-vl6x2","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:50Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:51Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ttl-initializing-vl6x2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"ttl-initializing-jj9hd","rayClusterName":"ttl-initializing-vl6x2","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:50Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:51Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ttl-initializing-vl6x2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:38:51.879Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"01ef4835-c9d8-44bf-b9d1-2fbbaf2b8d7d","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:38:51.880Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-initializing-vl6x2","reconcileID":"a41a07f0-074f-4d84-913b-75f976a20462","cluster":"ttl-initializing-vl6x2"}
{"level":"info","ts":"2026-11-03T15:38:51.881Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"6278ee6c-35f1-4266-8932-3b4e6d6e0e3b"}
{"level":"info","ts":"2026-11-03T15:38:51.881Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"6278ee6c-35f1-4266-8932-3b4e6d6e0e3b"}
{"level":"info","ts":"2026-11-03T15:38:51.881Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"6278ee6c-35f1-4266-8932-3b4e6d6e0e3b","1 head service found":"ttl-initializing-vl6x2-head-svc"}
{"level":"info","ts":"2026-11-03T15:38:51.881Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"6278ee6c-35f1-4266-8932-3b4e6d6e0e3b"}
{"level":"info","ts":"2026-11-03T15:38:51.931Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"6278ee6c-35f1-4266-8932-3b4e6d6e0e3b","error":"certificate ray-head-cert-ttl-initializing-vl6x2 is not ready at current generation 1"}
{"level":"error","ts":"2026-11-03T15:38:51.931Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"6278ee6c-35f1-4266-8932-3b4e6d6e0e3b","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-ttl-initializing-vl6x2 is not ready at current generation 1","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:38:51.931Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"6278ee6c-35f1-4266-8932-3b4e6d6e0e3b","error":"mTLS secrets not ready: certificate ray-head-cert-ttl-initializing-vl6x2 is not ready at current generation 1"}
{"level":"info","ts":"2026-11-03T15:38:51.931Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"6278ee6c-35f1-4266-8932-3b4e6d6e0e3b","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"ttl-initializing-vl6x2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:38:51.931Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"6278ee6c-35f1-4266-8932-3b4e6d6e0e3b","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"ttl-initializing-vl6x2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:38:51.932Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"6278ee6c-35f1-4266-8932-3b4e6d6e0e3b","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"ttl-initializing-vl6x2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:38:51.932Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"6278ee6c-35f1-4266-8932-3b4e6d6e0e3b"}
{"level":"error","ts":"2026-11-03T15:38:51.932Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"6278ee6c-35f1-4266-8932-3b4e6d6e0e3b","error":"mTLS secrets not ready: certificate ray-head-cert-ttl-initializing-vl6x2 is not ready at current generation 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:38:51.944Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"89c74803-e6a9-42cc-a68b-c5c9ed71ca5d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:38:51.944Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"89c74803-e6a9-42cc-a68b-c5c9ed71ca5d","name":"ttl-initializing-vl6x2"}
{"level":"info","ts":"2026-11-03T15:38:51.944Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"89c74803-e6a9-42cc-a68b-c5c9ed71ca5d","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:38:51.944Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"89c74803-e6a9-42cc-a68b-c5c9ed71ca5d"}
{"level":"info","ts":"2026-11-03T15:38:51.944Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"89c74803-e6a9-42cc-a68b-c5c9ed71ca5d","RayCluster":"ttl-initializing-vl6x2","State":""}
{"level":"info","ts":"2026-11-03T15:38:51.944Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"89c74803-e6a9-42cc-a68b-c5c9ed71ca5d","oldRayJobStatus":{"rayJobInfo":{},"jobId":"ttl-initializing-jj9hd","rayClusterName":"ttl-initializing-vl6x2","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:50Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:51Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ttl-initializing-vl6x2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"ttl-initializing-jj9hd","rayClusterName":"ttl-initializing-vl6x2","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:50Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:51Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ttl-initializing-vl6x2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:38:52.131Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"14bdc479-4bba-4dbc-a81d-4290c0a9e238","certificate":"ray-head-cert-ttl-initializing-vl6x2"}
{"level":"info","ts":"2026-11-03T15:38:52.131Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"14bdc479-4bba-4dbc-a81d-4290c0a9e238"}
{"level":"info","ts":"2026-11-03T15:38:52.131Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"08e22542-e797-4c92-afcf-57858eb15c6b","cluster":"ttl-initializing-vl6x2"}
{"level":"info","ts":"2026-11-03T15:38:52.252Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"8ca9e39b-7f81-4b0c-b49e-e4e144173188"}
{"level":"info","ts":"2026-11-03T15:38:52.252Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"8ca9e39b-7f81-4b0c-b49e-e4e144173188"}
{"level":"info","ts":"2026-11-03T15:38:52.252Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"8ca9e39b-7f81-4b0c-b49e-e4e144173188","1 head service found":"ttl-initializing-vl6x2-head-svc"}
{"level":"info","ts":"2026-11-03T15:38:52.252Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"8ca9e39b-7f81-4b0c-b49e-e4e144173188"}
{"level":"info","ts":"2026-11-03T15:38:52.305Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"8ca9e39b-7f81-4b0c-b49e-e4e144173188","error":"certificate ray-head-cert-ttl-initializing-vl6x2 is not ready at current generation 1"}
{"level":"error","ts":"2026-11-03T15:38:52.305Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"8ca9e39b-7f81-4b0c-b49e-e4e144173188","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-ttl-initializing-vl6x2 is not ready at current generation 1","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:38:52.305Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"8ca9e39b-7f81-4b0c-b49e-e4e144173188","error":"mTLS secrets not ready: certificate ray-head-cert-ttl-initializing-vl6x2 is not ready at current generation 1"}
{"level":"info","ts":"2026-11-03T15:38:52.305Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"8ca9e39b-7f81-4b0c-b49e-e4e144173188","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"ttl-initializing-vl6x2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:38:52.305Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"8ca9e39b-7f81-4b0c-b49e-e4e144173188","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"ttl-initializing-vl6x2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:38:52.305Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"8ca9e39b-7f81-4b0c-b49e-e4e144173188","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"ttl-initializing-vl6x2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:38:52.305Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"8ca9e39b-7f81-4b0c-b49e-e4e144173188"}
{"level":"error","ts":"2026-11-03T15:38:52.305Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"8ca9e39b-7f81-4b0c-b49e-e4e144173188","error":"mTLS secrets not ready: certificate ray-head-cert-ttl-initializing-vl6x2 is not ready at current generation 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:38:52.583Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"08e22542-e797-4c92-afcf-57858eb15c6b","certificate":"ray-head-cert-ttl-initializing-vl6x2"}
{"level":"info","ts":"2026-11-03T15:38:52.583Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"08e22542-e797-4c92-afcf-57858eb15c6b"}
{"level":"info","ts":"2026-11-03T15:38:52.946Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"1b1e3d0e-e1a2-4b12-9a9e-dbea2529da17"}
{"level":"info","ts":"2026-11-03T15:38:52.946Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"1b1e3d0e-e1a2-4b12-9a9e-dbea2529da17"}
{"level":"info","ts":"2026-11-03T15:38:52.946Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"1b1e3d0e-e1a2-4b12-9a9e-dbea2529da17","1 head service found":"ttl-initializing-vl6x2-head-svc"}
{"level":"info","ts":"2026-11-03T15:38:52.946Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"1b1e3d0e-e1a2-4b12-9a9e-dbea2529da17"}
{"level":"info","ts":"2026-11-03T15:38:52.996Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"1b1e3d0e-e1a2-4b12-9a9e-dbea2529da17","error":"certificate ray-head-cert-ttl-initializing-vl6x2 is not ready at current generation 1"}
{"level":"error","ts":"2026-11-03T15:38:52.996Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"1b1e3d0e-e1a2-4b12-9a9e-dbea2529da17","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-ttl-initializing-vl6x2 is not ready at current generation 1","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:38:52.996Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"1b1e3d0e-e1a2-4b12-9a9e-dbea2529da17","error":"mTLS secrets not ready: certificate ray-head-cert-ttl-initializing-vl6x2 is not ready at current generation 1"}
{"level":"info","ts":"2026-11-03T15:38:52.996Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"1b1e3d0e-e1a2-4b12-9a9e-dbea2529da17","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"ttl-initializing-vl6x2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:38:52.996Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"1b1e3d0e-e1a2-4b12-9a9e-dbea2529da17","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"ttl-initializing-vl6x2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:38:52.996Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"1b1e3d0e-e1a2-4b12-9a9e-dbea2529da17","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"ttl-initializing-vl6x2","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:38:52.996Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"1b1e3d0e-e1a2-4b12-9a9e-dbea2529da17"}
{"level":"error","ts":"2026-11-03T15:38:52.996Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"1b1e3d0e-e1a2-4b12-9a9e-dbea2529da17","error":"mTLS secrets not ready: certificate ray-head-cert-ttl-initializing-vl6x2 is not ready at current generation 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:38:54.277Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"d1689993-ee49-44d8-ab65-a15356cd28c2"}
{"level":"info","ts":"2026-11-03T15:38:54.277Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"d1689993-ee49-44d8-ab65-a15356cd28c2"}
{"level":"info","ts":"2026-11-03T15:38:54.277Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"d1689993-ee49-44d8-ab65-a15356cd28c2","1 head service found":"ttl-initializing-vl6x2-head-svc"}
{"level":"info","ts":"2026-11-03T15:38:54.277Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"d1689993-ee49-44d8-ab65-a15356cd28c2"}
{"level":"info","ts":"2026-11-03T15:38:54.378Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"d1689993-ee49-44d8-ab65-a15356cd28c2","mode":"IntegratedOAuth","cluster":"ttl-initializing-vl6x2"}
{"level":"info","ts":"2026-11-03T15:38:54.378Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"d1689993-ee49-44d8-ab65-a15356cd28c2","cluster":"ttl-initializing-vl6x2","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-11-03T15:38:54.378Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"d1689993-ee49-44d8-ab65-a15356cd28c2","cluster":"ttl-initializing-vl6x2","authType":"IntegratedOAuth","serviceAccount":"ttl-initializing-vl6x2-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-11-03T15:38:54.378Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"d1689993-ee49-44d8-ab65-a15356cd28c2","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"ttl-initializing-vl6x2","ray.io/group":"headgroup","ray.io/identifier":"ttl-initializing-vl6x2-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-11-03T15:38:54.378Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"d1689993-ee49-44d8-ab65-a15356cd28c2","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-11-03T15:38:54.378Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"d1689993-ee49-44d8-ab65-a15356cd28c2","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-11-03T15:38:54.378Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"d1689993-ee49-44d8-ab65-a15356cd28c2","rayNodeType":"head","generatedCmd":"ulimit -n ${RAY_START_ULIMIT_OPEN_FILES:-65536}; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-11-03T15:38:54.378Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"d1689993-ee49-44d8-ab65-a15356cd28c2","enabled":true}
{"level":"info","ts":"2026-11-03T15:38:54.440Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"d1689993-ee49-44d8-ab65-a15356cd28c2","name":"ttl-initializing-vl6x2-head-g65hj"}
{"level":"info","ts":"2026-11-03T15:38:54.440Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"d1689993-ee49-44d8-ab65-a15356cd28c2","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:38:54.440Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"d1689993-ee49-44d8-ab65-a15356cd28c2","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:38:54.440Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"d1689993-ee49-44d8-ab65-a15356cd28c2","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-11-03T15:38:54.440Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"d1689993-ee49-44d8-ab65-a15356cd28c2","group":"small-group","indices":{}}
{"level":"info","ts":"2026-11-03T15:38:54.440Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"d1689993-ee49-44d8-ab65-a15356cd28c2","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-11-03T15:38:54.540Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"d1689993-ee49-44d8-ab65-a15356cd28c2","creating worker for group":"small-group","index":0,"total":1,"replicaIndex":0}
{"level":"info","ts":"2026-11-03T15:38:54.540Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"d1689993-ee49-44d8-ab65-a15356cd28c2","nodeType":"worker","rayStartParams":{"address":"ttl-initializing-vl6x2-head-svc.test-ns-kbx4w.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-11-03T15:38:54.540Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"d1689993-ee49-44d8-ab65-a15356cd28c2","rayStartCmd":"ray start  --address=ttl-initializing-vl6x2-head-svc.test-ns-kbx4w.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-11-03T15:38:54.540Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"d1689993-ee49-44d8-ab65-a15356cd28c2","rayNodeType":"worker","generatedCmd":"ulimit -n ${RAY_START_ULIMIT_OPEN_FILES:-65536}; ray start  --address=ttl-initializing-vl6x2-head-svc.test-ns-kbx4w.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-11-03T15:38:54.540Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"d1689993-ee49-44d8-ab65-a15356cd28c2","enabled":true}
{"level":"info","ts":"2026-11-03T15:38:54.602Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"d1689993-ee49-44d8-ab65-a15356cd28c2","name":"ttl-initializing-vl6x2-small-group-worker-qsr7s"}
{"level":"info","ts":"2026-11-03T15:38:54.602Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"d1689993-ee49-44d8-ab65-a15356cd28c2","podName":"ttl-initializing-vl6x2-head-g65hj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:38:54.602Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"d1689993-ee49-44d8-ab65-a15356cd28c2","name":"ttl-initializing-vl6x2","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:51Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ttl-initializing-vl6x2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ttl-initializing-vl6x2-head-g65hj","serviceName":"ttl-initializing-vl6x2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:38:54.662Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"5d415cea-26f6-4d4f-9e4f-2e92fc8eedb9"}
{"level":"info","ts":"2026-11-03T15:38:54.662Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"5d415cea-26f6-4d4f-9e4f-2e92fc8eedb9"}
{"level":"info","ts":"2026-11-03T15:38:54.662Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"5d415cea-26f6-4d4f-9e4f-2e92fc8eedb9","1 head service found":"ttl-initializing-vl6x2-head-svc"}
{"level":"info","ts":"2026-11-03T15:38:54.662Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"365e5021-341e-4968-b5b2-667aede2b52f","cluster":"ttl-initializing-vl6x2"}
{"level":"info","ts":"2026-11-03T15:38:54.662Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"5d415cea-26f6-4d4f-9e4f-2e92fc8eedb9","Found 1 head Pod":"ttl-initializing-vl6x2-head-g65hj","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:38:54.662Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"5d415cea-26f6-4d4f-9e4f-2e92fc8eedb9","head Pod":"ttl-initializing-vl6x2-head-g65hj","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ttl-initializing-vl6x2-head-g65hj. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:38:54.662Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"5d415cea-26f6-4d4f-9e4f-2e92fc8eedb9","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:38:54.662Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"5d415cea-26f6-4d4f-9e4f-2e92fc8eedb9","worker Pod":"ttl-initializing-vl6x2-small-group-worker-qsr7s","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ttl-initializing-vl6x2-small-group-worker-qsr7s. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:38:54.662Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"5d415cea-26f6-4d4f-9e4f-2e92fc8eedb9","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:38:54.662Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"5d415cea-26f6-4d4f-9e4f-2e92fc8eedb9","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:38:54.662Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"5d415cea-26f6-4d4f-9e4f-2e92fc8eedb9","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:38:54.662Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"5d415cea-26f6-4d4f-9e4f-2e92fc8eedb9","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:38:54.662Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"e4f7c773-8849-48c7-8ed3-18ab71fcf1c6","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:38:54.662Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"e4f7c773-8849-48c7-8ed3-18ab71fcf1c6","name":"ttl-initializing-vl6x2"}
{"level":"info","ts":"2026-11-03T15:38:54.662Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"5d415cea-26f6-4d4f-9e4f-2e92fc8eedb9","podName":"ttl-initializing-vl6x2-head-g65hj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:38:54.662Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"e4f7c773-8849-48c7-8ed3-18ab71fcf1c6","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:38:54.662Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"5d415cea-26f6-4d4f-9e4f-2e92fc8eedb9","podName":"ttl-initializing-vl6x2-head-g65hj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:38:54.662Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"5d415cea-26f6-4d4f-9e4f-2e92fc8eedb9","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:38:54.662Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"5d415cea-26f6-4d4f-9e4f-2e92fc8eedb9","seconds":300}
{"level":"info","ts":"2026-11-03T15:38:54.662Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"e4f7c773-8849-48c7-8ed3-18ab71fcf1c6"}
{"level":"info","ts":"2026-11-03T15:38:54.662Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"e4f7c773-8849-48c7-8ed3-18ab71fcf1c6","RayCluster":"ttl-initializing-vl6x2","State":""}
{"level":"info","ts":"2026-11-03T15:38:54.662Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"e4f7c773-8849-48c7-8ed3-18ab71fcf1c6","oldRayJobStatus":{"rayJobInfo":{},"jobId":"ttl-initializing-jj9hd","rayClusterName":"ttl-initializing-vl6x2","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:50Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:51Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ttl-initializing-vl6x2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"ttl-initializing-jj9hd","rayClusterName":"ttl-initializing-vl6x2","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:50Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ttl-initializing-vl6x2-head-g65hj","serviceName":"ttl-initializing-vl6x2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:38:54.663Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"e4f7c773-8849-48c7-8ed3-18ab71fcf1c6","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:38:54.675Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"33d95adf-f03b-4efc-bf4f-5349382f1b40"}
{"level":"info","ts":"2026-11-03T15:38:54.675Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"33d95adf-f03b-4efc-bf4f-5349382f1b40"}
{"level":"info","ts":"2026-11-03T15:38:54.675Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"33d95adf-f03b-4efc-bf4f-5349382f1b40","1 head service found":"ttl-initializing-vl6x2-head-svc"}
{"level":"info","ts":"2026-11-03T15:38:54.675Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"33d95adf-f03b-4efc-bf4f-5349382f1b40","Found 1 head Pod":"ttl-initializing-vl6x2-head-g65hj","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:38:54.675Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"33d95adf-f03b-4efc-bf4f-5349382f1b40","head Pod":"ttl-initializing-vl6x2-head-g65hj","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ttl-initializing-vl6x2-head-g65hj. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:38:54.675Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"33d95adf-f03b-4efc-bf4f-5349382f1b40","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:38:54.675Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"33d95adf-f03b-4efc-bf4f-5349382f1b40","worker Pod":"ttl-initializing-vl6x2-small-group-worker-qsr7s","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ttl-initializing-vl6x2-small-group-worker-qsr7s. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:38:54.675Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"33d95adf-f03b-4efc-bf4f-5349382f1b40","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:38:54.675Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"33d95adf-f03b-4efc-bf4f-5349382f1b40","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:38:54.675Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"33d95adf-f03b-4efc-bf4f-5349382f1b40","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:38:54.675Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"33d95adf-f03b-4efc-bf4f-5349382f1b40","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:38:54.675Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"33d95adf-f03b-4efc-bf4f-5349382f1b40","podName":"ttl-initializing-vl6x2-head-g65hj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:38:54.675Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"33d95adf-f03b-4efc-bf4f-5349382f1b40","podName":"ttl-initializing-vl6x2-head-g65hj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:38:54.675Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"33d95adf-f03b-4efc-bf4f-5349382f1b40","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:38:54.675Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"33d95adf-f03b-4efc-bf4f-5349382f1b40","seconds":300}
{"level":"info","ts":"2026-11-03T15:38:54.722Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"3c42ad73-bcc5-4164-b32f-9f11ec04a055","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:38:54.722Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"3c42ad73-bcc5-4164-b32f-9f11ec04a055","name":"ttl-initializing-vl6x2"}
{"level":"info","ts":"2026-11-03T15:38:54.722Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"3c42ad73-bcc5-4164-b32f-9f11ec04a055","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:38:54.723Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"3c42ad73-bcc5-4164-b32f-9f11ec04a055"}
{"level":"info","ts":"2026-11-03T15:38:54.723Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"3c42ad73-bcc5-4164-b32f-9f11ec04a055","RayCluster":"ttl-initializing-vl6x2","State":""}
{"level":"info","ts":"2026-11-03T15:38:54.723Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"3c42ad73-bcc5-4164-b32f-9f11ec04a055","oldRayJobStatus":{"rayJobInfo":{},"jobId":"ttl-initializing-jj9hd","rayClusterName":"ttl-initializing-vl6x2","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:50Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ttl-initializing-vl6x2-head-g65hj","serviceName":"ttl-initializing-vl6x2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"ttl-initializing-jj9hd","rayClusterName":"ttl-initializing-vl6x2","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:50Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ttl-initializing-vl6x2-head-g65hj","serviceName":"ttl-initializing-vl6x2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:38:54.735Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"1056eb66-c2e2-487e-bafe-b1cedd694165"}
{"level":"info","ts":"2026-11-03T15:38:54.735Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"1056eb66-c2e2-487e-bafe-b1cedd694165"}
{"level":"info","ts":"2026-11-03T15:38:54.735Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"1056eb66-c2e2-487e-bafe-b1cedd694165","1 head service found":"ttl-initializing-vl6x2-head-svc"}
{"level":"info","ts":"2026-11-03T15:38:54.735Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"1056eb66-c2e2-487e-bafe-b1cedd694165","Found 1 head Pod":"ttl-initializing-vl6x2-head-g65hj","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:38:54.735Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"1056eb66-c2e2-487e-bafe-b1cedd694165","head Pod":"ttl-initializing-vl6x2-head-g65hj","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ttl-initializing-vl6x2-head-g65hj. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:38:54.735Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"1056eb66-c2e2-487e-bafe-b1cedd694165","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:38:54.735Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"1056eb66-c2e2-487e-bafe-b1cedd694165","worker Pod":"ttl-initializing-vl6x2-small-group-worker-qsr7s","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ttl-initializing-vl6x2-small-group-worker-qsr7s. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:38:54.735Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"1056eb66-c2e2-487e-bafe-b1cedd694165","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:38:54.735Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"1056eb66-c2e2-487e-bafe-b1cedd694165","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:38:54.735Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"1056eb66-c2e2-487e-bafe-b1cedd694165","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:38:54.735Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"1056eb66-c2e2-487e-bafe-b1cedd694165","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:38:54.736Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"1056eb66-c2e2-487e-bafe-b1cedd694165","podName":"ttl-initializing-vl6x2-head-g65hj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:38:54.736Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"1056eb66-c2e2-487e-bafe-b1cedd694165","podName":"ttl-initializing-vl6x2-head-g65hj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:38:54.736Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"1056eb66-c2e2-487e-bafe-b1cedd694165","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:38:54.736Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"1056eb66-c2e2-487e-bafe-b1cedd694165","seconds":300}
{"level":"info","ts":"2026-11-03T15:38:55.198Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"c798c6ef-76c0-4890-ae71-57d34888bb4f"}
{"level":"info","ts":"2026-11-03T15:38:55.198Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"c798c6ef-76c0-4890-ae71-57d34888bb4f"}
{"level":"info","ts":"2026-11-03T15:38:55.198Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"c798c6ef-76c0-4890-ae71-57d34888bb4f","1 head service found":"ttl-initializing-vl6x2-head-svc"}
{"level":"info","ts":"2026-11-03T15:38:55.198Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"c798c6ef-76c0-4890-ae71-57d34888bb4f","Found 1 head Pod":"ttl-initializing-vl6x2-head-g65hj","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:38:55.198Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"c798c6ef-76c0-4890-ae71-57d34888bb4f","head Pod":"ttl-initializing-vl6x2-head-g65hj","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ttl-initializing-vl6x2-head-g65hj. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:38:55.198Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"c798c6ef-76c0-4890-ae71-57d34888bb4f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:38:55.198Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"c798c6ef-76c0-4890-ae71-57d34888bb4f","worker Pod":"ttl-initializing-vl6x2-small-group-worker-qsr7s","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ttl-initializing-vl6x2-small-group-worker-qsr7s. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:38:55.198Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"c798c6ef-76c0-4890-ae71-57d34888bb4f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:38:55.198Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"c798c6ef-76c0-4890-ae71-57d34888bb4f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:38:55.198Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"c798c6ef-76c0-4890-ae71-57d34888bb4f","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:38:55.198Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"c798c6ef-76c0-4890-ae71-57d34888bb4f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:38:55.198Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"c798c6ef-76c0-4890-ae71-57d34888bb4f","podName":"ttl-initializing-vl6x2-small-group-worker-qsr7s","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:38:55.198Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"c798c6ef-76c0-4890-ae71-57d34888bb4f","podName":"ttl-initializing-vl6x2-small-group-worker-qsr7s","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:38:55.198Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"c798c6ef-76c0-4890-ae71-57d34888bb4f","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:38:55.198Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"c798c6ef-76c0-4890-ae71-57d34888bb4f","seconds":300}
{"level":"info","ts":"2026-11-03T15:38:55.710Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"189bac09-c9a8-4e7d-abee-a0e11d9e4b00"}
{"level":"info","ts":"2026-11-03T15:38:55.710Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"189bac09-c9a8-4e7d-abee-a0e11d9e4b00"}
{"level":"info","ts":"2026-11-03T15:38:55.710Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"189bac09-c9a8-4e7d-abee-a0e11d9e4b00","1 head service found":"ttl-initializing-vl6x2-head-svc"}
{"level":"info","ts":"2026-11-03T15:38:55.710Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"189bac09-c9a8-4e7d-abee-a0e11d9e4b00","Found 1 head Pod":"ttl-initializing-vl6x2-head-g65hj","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:38:55.710Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"189bac09-c9a8-4e7d-abee-a0e11d9e4b00","head Pod":"ttl-initializing-vl6x2-head-g65hj","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ttl-initializing-vl6x2-head-g65hj. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:38:55.710Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"189bac09-c9a8-4e7d-abee-a0e11d9e4b00","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:38:55.710Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"189bac09-c9a8-4e7d-abee-a0e11d9e4b00","worker Pod":"ttl-initializing-vl6x2-small-group-worker-qsr7s","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ttl-initializing-vl6x2-small-group-worker-qsr7s. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:38:55.710Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"189bac09-c9a8-4e7d-abee-a0e11d9e4b00","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:38:55.710Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"189bac09-c9a8-4e7d-abee-a0e11d9e4b00","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:38:55.710Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"189bac09-c9a8-4e7d-abee-a0e11d9e4b00","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:38:55.710Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"189bac09-c9a8-4e7d-abee-a0e11d9e4b00","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:38:55.710Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"189bac09-c9a8-4e7d-abee-a0e11d9e4b00","podName":"ttl-initializing-vl6x2-head-g65hj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:38:55.710Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"189bac09-c9a8-4e7d-abee-a0e11d9e4b00","podName":"ttl-initializing-vl6x2-head-g65hj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:38:55.710Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"189bac09-c9a8-4e7d-abee-a0e11d9e4b00","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:38:55.710Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"189bac09-c9a8-4e7d-abee-a0e11d9e4b00","seconds":300}
{"level":"info","ts":"2026-11-03T15:38:56.062Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"1cc36048-75a9-461e-a501-07c516c1a6ac"}
{"level":"info","ts":"2026-11-03T15:38:56.062Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"1cc36048-75a9-461e-a501-07c516c1a6ac"}
{"level":"info","ts":"2026-11-03T15:38:56.062Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"1cc36048-75a9-461e-a501-07c516c1a6ac","1 head service found":"ttl-initializing-vl6x2-head-svc"}
{"level":"info","ts":"2026-11-03T15:38:56.063Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"1cc36048-75a9-461e-a501-07c516c1a6ac","Found 1 head Pod":"ttl-initializing-vl6x2-head-g65hj","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:38:56.063Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"1cc36048-75a9-461e-a501-07c516c1a6ac","head Pod":"ttl-initializing-vl6x2-head-g65hj","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ttl-initializing-vl6x2-head-g65hj. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:38:56.063Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"1cc36048-75a9-461e-a501-07c516c1a6ac","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:38:56.063Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"1cc36048-75a9-461e-a501-07c516c1a6ac","worker Pod":"ttl-initializing-vl6x2-small-group-worker-qsr7s","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ttl-initializing-vl6x2-small-group-worker-qsr7s. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:38:56.063Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"1cc36048-75a9-461e-a501-07c516c1a6ac","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:38:56.063Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"1cc36048-75a9-461e-a501-07c516c1a6ac","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:38:56.063Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"1cc36048-75a9-461e-a501-07c516c1a6ac","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:38:56.063Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"1cc36048-75a9-461e-a501-07c516c1a6ac","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:38:56.063Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"1cc36048-75a9-461e-a501-07c516c1a6ac","podName":"ttl-initializing-vl6x2-head-g65hj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:38:56.063Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"1cc36048-75a9-461e-a501-07c516c1a6ac","podName":"ttl-initializing-vl6x2-head-g65hj","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:38:56.063Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"1cc36048-75a9-461e-a501-07c516c1a6ac","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:38:56.063Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"1cc36048-75a9-461e-a501-07c516c1a6ac","seconds":300}
{"level":"info","ts":"2026-11-03T15:38:57.723Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"174a8a53-a540-4d6a-80e8-554915fe5bda","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:38:57.723Z","logger":"controllers.RayJob","msg":"The RayJob has passed the preRunningDeadlineSeconds. Transition the status to `Failed`.","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"174a8a53-a540-4d6a-80e8-554915fe5bda","StartTime":"2026-11-03 15:38:50 +0000 UTC","PreRunningDeadlineSeconds":5}
{"level":"info","ts":"2026-11-03T15:38:57.723Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"174a8a53-a540-4d6a-80e8-554915fe5bda","oldRayJobStatus":{"rayJobInfo":{},"jobId":"ttl-initializing-jj9hd","rayClusterName":"ttl-initializing-vl6x2","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:50Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ttl-initializing-vl6x2-head-g65hj","serviceName":"ttl-initializing-vl6x2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"ttl-initializing-jj9hd","rayClusterName":"ttl-initializing-vl6x2","jobDeploymentStatus":"Failed","reason":"PreRunningDeadlineExceeded","message":"The RayJob has passed the preRunningDeadlineSeconds. StartTime: 2026-11-03 15:38:50 +0000 UTC. PreRunningDeadlineSeconds: 5","startTime":"2026-11-03T15:38:50Z","succeeded":0,"failed":1,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ttl-initializing-vl6x2-head-g65hj","serviceName":"ttl-initializing-vl6x2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:38:57.723Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"174a8a53-a540-4d6a-80e8-554915fe5bda","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Failed"}
{"level":"info","ts":"2026-11-03T15:38:57.820Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"1b6d577b-a886-49c2-b602-dd1ed2d5ae73","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:38:57.820Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"1b6d577b-a886-49c2-b602-dd1ed2d5ae73","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Failed","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-11-03 15:38:57 +0000 UTC","Now":"2026-11-03T15:38:57.820Z","ShutdownTime":"2026-11-03T15:38:57.000Z"}
{"level":"info","ts":"2026-11-03T15:38:57.832Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"c0297090-8500-45ab-bb37-0af26959c799"}
{"level":"info","ts":"2026-11-03T15:38:57.832Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"c0297090-8500-45ab-bb37-0af26959c799"}
{"level":"info","ts":"2026-11-03T15:38:57.832Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"c0297090-8500-45ab-bb37-0af26959c799","1 head service found":"ttl-initializing-vl6x2-head-svc"}
{"level":"info","ts":"2026-11-03T15:38:57.832Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"c0297090-8500-45ab-bb37-0af26959c799","Found 1 head Pod":"ttl-initializing-vl6x2-head-g65hj","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:38:57.832Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"c0297090-8500-45ab-bb37-0af26959c799","head Pod":"ttl-initializing-vl6x2-head-g65hj","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ttl-initializing-vl6x2-head-g65hj. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:38:57.832Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"c0297090-8500-45ab-bb37-0af26959c799","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:38:57.832Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"c0297090-8500-45ab-bb37-0af26959c799","worker Pod":"ttl-initializing-vl6x2-small-group-worker-qsr7s","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ttl-initializing-vl6x2-small-group-worker-qsr7s. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:38:57.832Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"c0297090-8500-45ab-bb37-0af26959c799","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:38:57.832Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"c0297090-8500-45ab-bb37-0af26959c799","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:38:57.832Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"c0297090-8500-45ab-bb37-0af26959c799","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:38:57.832Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"c0297090-8500-45ab-bb37-0af26959c799","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:38:57.832Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"c0297090-8500-45ab-bb37-0af26959c799","podName":"ttl-initializing-vl6x2-small-group-worker-qsr7s","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:38:57.832Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"c0297090-8500-45ab-bb37-0af26959c799","podName":"ttl-initializing-vl6x2-small-group-worker-qsr7s","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:38:57.833Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"c0297090-8500-45ab-bb37-0af26959c799","name":"ttl-initializing-vl6x2","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ttl-initializing-vl6x2-head-g65hj","serviceName":"ttl-initializing-vl6x2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:57Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.50","serviceIP":"10.132.0.50","podName":"ttl-initializing-vl6x2-head-g65hj","serviceName":"ttl-initializing-vl6x2-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:38:57.879Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob is deleted","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"1b6d577b-a886-49c2-b602-dd1ed2d5ae73","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:38:57.879Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"1b6d577b-a886-49c2-b602-dd1ed2d5ae73","isClusterDeleted":false}
{"level":"info","ts":"2026-11-03T15:38:57.879Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"1b6d577b-a886-49c2-b602-dd1ed2d5ae73","deletionMechanism":"ShutdownAfterJobFinishes","RayCluster":"ttl-initializing-vl6x2"}
{"level":"info","ts":"2026-11-03T15:38:57.879Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-initializing-vl6x2","reconcileID":"2f4ced1a-5f36-41e8-9b32-051358827698","namespacedName":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:38:57.879Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-initializing-vl6x2","reconcileID":"2f4ced1a-5f36-41e8-9b32-051358827698","cluster":"ttl-initializing-vl6x2"}
{"level":"info","ts":"2026-11-03T15:38:57.879Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-initializing-vl6x2","reconcileID":"2f4ced1a-5f36-41e8-9b32-051358827698","cluster":"ttl-initializing-vl6x2"}
{"level":"info","ts":"2026-11-03T15:38:57.879Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"03f0b59c-c94f-47b1-8eb0-4add1e97d1da","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:38:57.879Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"03f0b59c-c94f-47b1-8eb0-4add1e97d1da","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Failed","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-11-03 15:38:57 +0000 UTC","Now":"2026-11-03T15:38:57.879Z","ShutdownTime":"2026-11-03T15:38:57.000Z"}
{"level":"info","ts":"2026-11-03T15:38:57.880Z","logger":"controllers.RayJob","msg":"The deletion of the associated RayCluster for RayJob is ongoing.","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"03f0b59c-c94f-47b1-8eb0-4add1e97d1da","RayCluster":"ttl-initializing-vl6x2"}
{"level":"info","ts":"2026-11-03T15:38:57.880Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"03f0b59c-c94f-47b1-8eb0-4add1e97d1da","isClusterDeleted":false}
{"level":"info","ts":"2026-11-03T15:38:57.880Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"03f0b59c-c94f-47b1-8eb0-4add1e97d1da","deletionMechanism":"ShutdownAfterJobFinishes","RayCluster":"ttl-initializing-vl6x2"}
{"level":"info","ts":"2026-11-03T15:38:57.879Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-initializing-vl6x2","reconcileID":"2f4ced1a-5f36-41e8-9b32-051358827698","namespace":"test-ns-kbx4w","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-11-03T15:38:57.887Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"c0297090-8500-45ab-bb37-0af26959c799","name":"ttl-initializing-vl6x2","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ttl-initializing-vl6x2\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-kbx4w","name":"ttl-initializing-vl6x2"}}
{"level":"info","ts":"2026-11-03T15:38:57.887Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"c0297090-8500-45ab-bb37-0af26959c799"}
{"level":"error","ts":"2026-11-03T15:38:57.887Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"c0297090-8500-45ab-bb37-0af26959c799","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ttl-initializing-vl6x2\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:38:57.887Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"reconcileID":"afdad8f1-dbb3-49dd-8162-f6589b33f3df"}
{"level":"info","ts":"2026-11-03T15:38:57.931Z","logger":"authentication-controller","msg":"Deleted shared ReferenceGrant (last cluster with auth)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-initializing-vl6x2","reconcileID":"2f4ced1a-5f36-41e8-9b32-051358827698","name":"kuberay-gateway-access","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:38:57.983Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-initializing-vl6x2","reconcileID":"2f4ced1a-5f36-41e8-9b32-051358827698","name":"test-ns-kbx4w-ttl-initializing-vl6x2","namespace":"opendatahub"}
{"level":"info","ts":"2026-11-03T15:38:58.037Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-initializing-vl6x2","reconcileID":"2f4ced1a-5f36-41e8-9b32-051358827698","configMap":"kube-rbac-proxy-config-ttl-initializing-vl6x2"}
{"level":"info","ts":"2026-11-03T15:38:58.063Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"a4d7b07d-b12b-4744-a40b-f75299b2ce9d","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-11-03T15:38:58.063Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"a4d7b07d-b12b-4744-a40b-f75299b2ce9d","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-11-03T15:38:58.089Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-initializing-vl6x2","reconcileID":"2f4ced1a-5f36-41e8-9b32-051358827698","serviceAccount":"ttl-initializing-vl6x2-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-11-03T15:38:58.120Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"a4d7b07d-b12b-4744-a40b-f75299b2ce9d"}
{"level":"info","ts":"2026-11-03T15:38:58.120Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"a4d7b07d-b12b-4744-a40b-f75299b2ce9d","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""}
{"level":"info","ts":"2026-11-03T15:38:58.120Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"a4d7b07d-b12b-4744-a40b-f75299b2ce9d","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:38:58.120Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"a4d7b07d-b12b-4744-a40b-f75299b2ce9d","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:38:58.148Z","logger":"raycluster-resource","msg":"default","name":"ttl-initializing-vl6x2"}
{"level":"info","ts":"2026-11-03T15:38:58.148Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:38:58.178Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"7c48c6ed-1986-4ecf-ab4f-e6126e6a4689","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-11-03T15:38:58.178Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"7c48c6ed-1986-4ecf-ab4f-e6126e6a4689","name":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:38:58.178Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"7c48c6ed-1986-4ecf-ab4f-e6126e6a4689","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:38:58.178Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"7c48c6ed-1986-4ecf-ab4f-e6126e6a4689","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:38:58.204Z","logger":"raycluster-resource","msg":"validate update","name":"ttl-initializing-vl6x2"}
{"level":"info","ts":"2026-11-03T15:38:58.233Z","logger":"raycluster-resource","msg":"default","name":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:38:58.234Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:38:58.264Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-initializing-vl6x2","reconcileID":"2f4ced1a-5f36-41e8-9b32-051358827698","cluster":"ttl-initializing-vl6x2"}
{"level":"info","ts":"2026-11-03T15:38:58.264Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-initializing-vl6x2","reconcileID":"efa0cc50-e70a-4267-9cd3-8c8e2b572a12","namespacedName":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:38:58.264Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-initializing-vl6x2","reconcileID":"efa0cc50-e70a-4267-9cd3-8c8e2b572a12"}
{"level":"info","ts":"2026-11-03T15:38:58.264Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-initializing-vl6x2","reconcileID":"efa0cc50-e70a-4267-9cd3-8c8e2b572a12","namespace":"test-ns-kbx4w","authClusters":0}
{"level":"info","ts":"2026-11-03T15:38:58.264Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-initializing-vl6x2","reconcileID":"efa0cc50-e70a-4267-9cd3-8c8e2b572a12","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:38:58.298Z","logger":"raycluster-resource","msg":"validate create","name":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:38:58.299Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-initializing-vl6x2","reconcileID":"bea9fe10-89b2-4c41-b1f8-f2c2ddd924c3","namespacedName":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:38:58.299Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-initializing-vl6x2","reconcileID":"bea9fe10-89b2-4c41-b1f8-f2c2ddd924c3"}
{"level":"info","ts":"2026-11-03T15:38:58.299Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-initializing-vl6x2","reconcileID":"bea9fe10-89b2-4c41-b1f8-f2c2ddd924c3","namespace":"test-ns-kbx4w","authClusters":0}
{"level":"info","ts":"2026-11-03T15:38:58.299Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-initializing-vl6x2","reconcileID":"bea9fe10-89b2-4c41-b1f8-f2c2ddd924c3","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:38:58.351Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"7c48c6ed-1986-4ecf-ab4f-e6126e6a4689","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:38:58.351Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-waiting-kjf64","reconcileID":"731eeb33-3f28-4018-bdc3-49e4e55d006e","namespacedName":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:38:58.351Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-waiting-kjf64","reconcileID":"731eeb33-3f28-4018-bdc3-49e4e55d006e","mode":"IntegratedOAuth","cluster":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:38:58.351Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-waiting-kjf64","reconcileID":"731eeb33-3f28-4018-bdc3-49e4e55d006e","cluster":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:38:58.351Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-waiting-kjf64","reconcileID":"731eeb33-3f28-4018-bdc3-49e4e55d006e","cluster":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:38:58.351Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"7c48c6ed-1986-4ecf-ab4f-e6126e6a4689"}
{"level":"info","ts":"2026-11-03T15:38:58.351Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"7c48c6ed-1986-4ecf-ab4f-e6126e6a4689","RayCluster":"ttl-waiting-kjf64","State":""}
{"level":"info","ts":"2026-11-03T15:38:58.351Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"7c48c6ed-1986-4ecf-ab4f-e6126e6a4689","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:38:58.352Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"668b98a2-217c-4466-9999-07dc54e2d9cd","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-11-03T15:38:58.352Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"668b98a2-217c-4466-9999-07dc54e2d9cd","name":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:38:58.352Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"668b98a2-217c-4466-9999-07dc54e2d9cd","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:38:58.352Z","logger":"controllers.NetworkPolicy","msg":"Reconciling NetworkPolicies for RayCluster","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"dc730e06-01ae-4759-8a7e-db73e506d74b","cluster":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:38:58.352Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"668b98a2-217c-4466-9999-07dc54e2d9cd"}
{"level":"info","ts":"2026-11-03T15:38:58.352Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"668b98a2-217c-4466-9999-07dc54e2d9cd","RayCluster":"ttl-waiting-kjf64","State":""}
{"level":"info","ts":"2026-11-03T15:38:58.352Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"668b98a2-217c-4466-9999-07dc54e2d9cd","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:38:58.352Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"238f7e29-b305-46cb-91bf-c1a76e52f04f","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:38:58.352Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"238f7e29-b305-46cb-91bf-c1a76e52f04f","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Failed","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-11-03 15:38:57 +0000 UTC","Now":"2026-11-03T15:38:58.352Z","ShutdownTime":"2026-11-03T15:38:57.000Z"}
{"level":"info","ts":"2026-11-03T15:38:58.352Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"238f7e29-b305-46cb-91bf-c1a76e52f04f","RayCluster":{"name":"ttl-initializing-vl6x2","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:38:58.352Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"238f7e29-b305-46cb-91bf-c1a76e52f04f","isClusterDeleted":true}
{"level":"info","ts":"2026-11-03T15:38:58.352Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"238f7e29-b305-46cb-91bf-c1a76e52f04f","deletionMechanism":"ShutdownAfterJobFinishes","RayCluster":"ttl-initializing-vl6x2"}
{"level":"info","ts":"2026-11-03T15:38:58.352Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"bc1cf08a-5c3c-4af1-9088-e73ad70753df"}
{"level":"info","ts":"2026-11-03T15:38:58.352Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"bc1cf08a-5c3c-4af1-9088-e73ad70753df"}
{"level":"info","ts":"2026-11-03T15:38:58.352Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"c3d38892-343a-443f-86af-a11656818c53","cluster":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:38:58.352Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"380ea387-b5b6-4a7d-bc5f-b286b9516b08","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-11-03T15:38:58.352Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"380ea387-b5b6-4a7d-bc5f-b286b9516b08","name":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:38:58.352Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"380ea387-b5b6-4a7d-bc5f-b286b9516b08","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:38:58.352Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"380ea387-b5b6-4a7d-bc5f-b286b9516b08"}
{"level":"info","ts":"2026-11-03T15:38:58.352Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"380ea387-b5b6-4a7d-bc5f-b286b9516b08","RayCluster":"ttl-waiting-kjf64","State":""}
{"level":"info","ts":"2026-11-03T15:38:58.352Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"380ea387-b5b6-4a7d-bc5f-b286b9516b08","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}}
{"level":"info","ts":"2026-11-03T15:38:58.408Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"bc1cf08a-5c3c-4af1-9088-e73ad70753df","name":"ttl-waiting-kjf64-head-svc"}
{"level":"info","ts":"2026-11-03T15:38:58.408Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"bc1cf08a-5c3c-4af1-9088-e73ad70753df"}
{"level":"info","ts":"2026-11-03T15:38:58.408Z","logger":"controllers.NetworkPolicy","msg":"Successfully created NetworkPolicy","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"dc730e06-01ae-4759-8a7e-db73e506d74b","name":"ttl-waiting-kjf64-head"}
{"level":"info","ts":"2026-11-03T15:38:58.409Z","logger":"raycluster-resource","msg":"default","name":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:38:58.409Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:38:58.458Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"bc1cf08a-5c3c-4af1-9088-e73ad70753df","error":"certificate ray-head-cert-ttl-waiting-kjf64 not found: certificates.cert-manager.io \"ray-head-cert-ttl-waiting-kjf64\" not found"}
{"level":"error","ts":"2026-11-03T15:38:58.458Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"bc1cf08a-5c3c-4af1-9088-e73ad70753df","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-ttl-waiting-kjf64 not found: certificates.cert-manager.io \"ray-head-cert-ttl-waiting-kjf64\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:38:58.458Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"bc1cf08a-5c3c-4af1-9088-e73ad70753df","error":"mTLS secrets not ready: certificate ray-head-cert-ttl-waiting-kjf64 not found: certificates.cert-manager.io \"ray-head-cert-ttl-waiting-kjf64\" not found"}
{"level":"info","ts":"2026-11-03T15:38:58.458Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"bc1cf08a-5c3c-4af1-9088-e73ad70753df","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"ttl-waiting-kjf64","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:38:58.459Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"bc1cf08a-5c3c-4af1-9088-e73ad70753df","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"ttl-waiting-kjf64","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:38:58.459Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"bc1cf08a-5c3c-4af1-9088-e73ad70753df","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"ttl-waiting-kjf64","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:38:58.459Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"bc1cf08a-5c3c-4af1-9088-e73ad70753df","name":"ttl-waiting-kjf64","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:38:58.460Z","logger":"controllers.NetworkPolicy","msg":"Successfully created NetworkPolicy","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"dc730e06-01ae-4759-8a7e-db73e506d74b","name":"ttl-waiting-kjf64-workers-small-group"}
{"level":"info","ts":"2026-11-03T15:38:58.460Z","logger":"controllers.NetworkPolicy","msg":"Successfully reconciled NetworkPolicies for RayCluster","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"dc730e06-01ae-4759-8a7e-db73e506d74b","cluster":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:38:58.460Z","logger":"controllers.NetworkPolicy","msg":"Reconciling NetworkPolicies for RayCluster","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"59904ea0-c446-46b0-8150-4e90cbb0e720","cluster":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:38:58.559Z","logger":"raycluster-resource","msg":"validate update","name":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:38:58.560Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"aaab179a-cb3b-42c2-bf6d-3d259e1b55f9","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-11-03T15:38:58.560Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"aaab179a-cb3b-42c2-bf6d-3d259e1b55f9","name":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:38:58.560Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"aaab179a-cb3b-42c2-bf6d-3d259e1b55f9","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:38:58.561Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"bc1cf08a-5c3c-4af1-9088-e73ad70753df"}
{"level":"error","ts":"2026-11-03T15:38:58.561Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"bc1cf08a-5c3c-4af1-9088-e73ad70753df","error":"mTLS secrets not ready: certificate ray-head-cert-ttl-waiting-kjf64 not found: certificates.cert-manager.io \"ray-head-cert-ttl-waiting-kjf64\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:38:58.561Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"ac44458d-d7ac-4592-b4ce-30a2ae7c0661"}
{"level":"info","ts":"2026-11-03T15:38:58.561Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"ac44458d-d7ac-4592-b4ce-30a2ae7c0661"}
{"level":"info","ts":"2026-11-03T15:38:58.561Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"aaab179a-cb3b-42c2-bf6d-3d259e1b55f9"}
{"level":"info","ts":"2026-11-03T15:38:58.561Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"ac44458d-d7ac-4592-b4ce-30a2ae7c0661","1 head service found":"ttl-waiting-kjf64-head-svc"}
{"level":"info","ts":"2026-11-03T15:38:58.561Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"aaab179a-cb3b-42c2-bf6d-3d259e1b55f9","RayCluster":"ttl-waiting-kjf64","State":""}
{"level":"info","ts":"2026-11-03T15:38:58.561Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"aaab179a-cb3b-42c2-bf6d-3d259e1b55f9","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:38:58.561Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"aaab179a-cb3b-42c2-bf6d-3d259e1b55f9","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:38:58.561Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"ac44458d-d7ac-4592-b4ce-30a2ae7c0661"}
{"level":"info","ts":"2026-11-03T15:38:58.575Z","logger":"controllers.NetworkPolicy","msg":"Successfully reconciled NetworkPolicies for RayCluster","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"59904ea0-c446-46b0-8150-4e90cbb0e720","cluster":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:38:58.612Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"ac44458d-d7ac-4592-b4ce-30a2ae7c0661","error":"certificate ray-head-cert-ttl-waiting-kjf64 not found: certificates.cert-manager.io \"ray-head-cert-ttl-waiting-kjf64\" not found"}
{"level":"error","ts":"2026-11-03T15:38:58.612Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"ac44458d-d7ac-4592-b4ce-30a2ae7c0661","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-ttl-waiting-kjf64 not found: certificates.cert-manager.io \"ray-head-cert-ttl-waiting-kjf64\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:38:58.612Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"ac44458d-d7ac-4592-b4ce-30a2ae7c0661","error":"mTLS secrets not ready: certificate ray-head-cert-ttl-waiting-kjf64 not found: certificates.cert-manager.io \"ray-head-cert-ttl-waiting-kjf64\" not found"}
{"level":"info","ts":"2026-11-03T15:38:58.612Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"ac44458d-d7ac-4592-b4ce-30a2ae7c0661","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"ttl-waiting-kjf64","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:38:58.612Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"ac44458d-d7ac-4592-b4ce-30a2ae7c0661","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"ttl-waiting-kjf64","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:38:58.612Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"ac44458d-d7ac-4592-b4ce-30a2ae7c0661","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"ttl-waiting-kjf64","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:38:58.612Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"ac44458d-d7ac-4592-b4ce-30a2ae7c0661"}
{"level":"error","ts":"2026-11-03T15:38:58.612Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"ac44458d-d7ac-4592-b4ce-30a2ae7c0661","error":"mTLS secrets not ready: certificate ray-head-cert-ttl-waiting-kjf64 not found: certificates.cert-manager.io \"ray-head-cert-ttl-waiting-kjf64\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:38:58.620Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"ca3d7492-b604-4c98-beda-28cc801be382","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-11-03T15:38:58.620Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"ca3d7492-b604-4c98-beda-28cc801be382","name":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:38:58.620Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"ca3d7492-b604-4c98-beda-28cc801be382","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:38:58.621Z","logger":"raycluster-resource","msg":"default","name":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:38:58.658Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"735c7fcc-b04f-4723-8665-5777d82d03e2"}
{"level":"info","ts":"2026-11-03T15:38:58.658Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"735c7fcc-b04f-4723-8665-5777d82d03e2"}
{"level":"info","ts":"2026-11-03T15:38:58.658Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"ca3d7492-b604-4c98-beda-28cc801be382"}
{"level":"info","ts":"2026-11-03T15:38:58.658Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"ca3d7492-b604-4c98-beda-28cc801be382","RayCluster":"ttl-waiting-kjf64","State":""}
{"level":"info","ts":"2026-11-03T15:38:58.658Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"735c7fcc-b04f-4723-8665-5777d82d03e2","1 head service found":"ttl-waiting-kjf64-head-svc"}
{"level":"info","ts":"2026-11-03T15:38:58.658Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"ca3d7492-b604-4c98-beda-28cc801be382","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:38:58.658Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:38:58.659Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"735c7fcc-b04f-4723-8665-5777d82d03e2"}
{"level":"error","ts":"2026-11-03T15:38:58.707Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-waiting-kjf64","reconcileID":"731eeb33-3f28-4018-bdc3-49e4e55d006e","error":"failed to add finalizer: Operation cannot be fulfilled on rayclusters.ray.io \"ttl-waiting-kjf64\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:167\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:38:58.707Z","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-waiting-kjf64","reconcileID":"731eeb33-3f28-4018-bdc3-49e4e55d006e"}
{"level":"error","ts":"2026-11-03T15:38:58.707Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-waiting-kjf64","reconcileID":"731eeb33-3f28-4018-bdc3-49e4e55d006e","error":"failed to add finalizer: Operation cannot be fulfilled on rayclusters.ray.io \"ttl-waiting-kjf64\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:38:58.707Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-waiting-kjf64","reconcileID":"1381edd1-b17e-47f2-963b-82f7fe195bea","namespacedName":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:38:58.707Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-waiting-kjf64","reconcileID":"1381edd1-b17e-47f2-963b-82f7fe195bea","mode":"IntegratedOAuth","cluster":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:38:58.707Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-waiting-kjf64","reconcileID":"1381edd1-b17e-47f2-963b-82f7fe195bea","cluster":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:38:58.707Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-waiting-kjf64","reconcileID":"1381edd1-b17e-47f2-963b-82f7fe195bea","cluster":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:38:58.708Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"735c7fcc-b04f-4723-8665-5777d82d03e2","error":"certificate ray-head-cert-ttl-waiting-kjf64 not found: certificates.cert-manager.io \"ray-head-cert-ttl-waiting-kjf64\" not found"}
{"level":"error","ts":"2026-11-03T15:38:58.708Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"735c7fcc-b04f-4723-8665-5777d82d03e2","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-ttl-waiting-kjf64 not found: certificates.cert-manager.io \"ray-head-cert-ttl-waiting-kjf64\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:38:58.708Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"735c7fcc-b04f-4723-8665-5777d82d03e2","error":"mTLS secrets not ready: certificate ray-head-cert-ttl-waiting-kjf64 not found: certificates.cert-manager.io \"ray-head-cert-ttl-waiting-kjf64\" not found"}
{"level":"info","ts":"2026-11-03T15:38:58.708Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"735c7fcc-b04f-4723-8665-5777d82d03e2","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"ttl-waiting-kjf64","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:38:58.708Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"735c7fcc-b04f-4723-8665-5777d82d03e2","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"ttl-waiting-kjf64","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:38:58.708Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"735c7fcc-b04f-4723-8665-5777d82d03e2","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"ttl-waiting-kjf64","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:38:58.708Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"735c7fcc-b04f-4723-8665-5777d82d03e2"}
{"level":"error","ts":"2026-11-03T15:38:58.708Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"735c7fcc-b04f-4723-8665-5777d82d03e2","error":"mTLS secrets not ready: certificate ray-head-cert-ttl-waiting-kjf64 not found: certificates.cert-manager.io \"ray-head-cert-ttl-waiting-kjf64\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:38:58.729Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"efc76650-4a07-4121-9ab6-1df47d63f228"}
{"level":"info","ts":"2026-11-03T15:38:58.729Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"efc76650-4a07-4121-9ab6-1df47d63f228"}
{"level":"info","ts":"2026-11-03T15:38:58.729Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"efc76650-4a07-4121-9ab6-1df47d63f228","1 head service found":"ttl-waiting-kjf64-head-svc"}
{"level":"info","ts":"2026-11-03T15:38:58.729Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"efc76650-4a07-4121-9ab6-1df47d63f228"}
{"level":"info","ts":"2026-11-03T15:38:58.766Z","logger":"raycluster-resource","msg":"default","name":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:38:58.766Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:38:58.778Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"efc76650-4a07-4121-9ab6-1df47d63f228","error":"certificate ray-head-cert-ttl-waiting-kjf64 not found: certificates.cert-manager.io \"ray-head-cert-ttl-waiting-kjf64\" not found"}
{"level":"error","ts":"2026-11-03T15:38:58.778Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"efc76650-4a07-4121-9ab6-1df47d63f228","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-ttl-waiting-kjf64 not found: certificates.cert-manager.io \"ray-head-cert-ttl-waiting-kjf64\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:38:58.778Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"efc76650-4a07-4121-9ab6-1df47d63f228","error":"mTLS secrets not ready: certificate ray-head-cert-ttl-waiting-kjf64 not found: certificates.cert-manager.io \"ray-head-cert-ttl-waiting-kjf64\" not found"}
{"level":"info","ts":"2026-11-03T15:38:58.778Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"efc76650-4a07-4121-9ab6-1df47d63f228","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"ttl-waiting-kjf64","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:38:58.778Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"efc76650-4a07-4121-9ab6-1df47d63f228","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"ttl-waiting-kjf64","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:38:58.778Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"efc76650-4a07-4121-9ab6-1df47d63f228","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"ttl-waiting-kjf64","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:38:58.778Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"efc76650-4a07-4121-9ab6-1df47d63f228"}
{"level":"error","ts":"2026-11-03T15:38:58.778Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"efc76650-4a07-4121-9ab6-1df47d63f228","error":"mTLS secrets not ready: certificate ray-head-cert-ttl-waiting-kjf64 not found: certificates.cert-manager.io \"ray-head-cert-ttl-waiting-kjf64\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:38:58.819Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"f1ef1e5e-14a0-4c88-927c-8f19d3fc1768"}
{"level":"info","ts":"2026-11-03T15:38:58.819Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"f1ef1e5e-14a0-4c88-927c-8f19d3fc1768"}
{"level":"info","ts":"2026-11-03T15:38:58.819Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"f1ef1e5e-14a0-4c88-927c-8f19d3fc1768","1 head service found":"ttl-waiting-kjf64-head-svc"}
{"level":"info","ts":"2026-11-03T15:38:58.819Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"f1ef1e5e-14a0-4c88-927c-8f19d3fc1768"}
{"level":"info","ts":"2026-11-03T15:38:58.822Z","logger":"raycluster-resource","msg":"validate update","name":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:38:58.869Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"f1ef1e5e-14a0-4c88-927c-8f19d3fc1768","error":"certificate ray-head-cert-ttl-waiting-kjf64 not found: certificates.cert-manager.io \"ray-head-cert-ttl-waiting-kjf64\" not found"}
{"level":"error","ts":"2026-11-03T15:38:58.869Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"f1ef1e5e-14a0-4c88-927c-8f19d3fc1768","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-ttl-waiting-kjf64 not found: certificates.cert-manager.io \"ray-head-cert-ttl-waiting-kjf64\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:38:58.869Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"f1ef1e5e-14a0-4c88-927c-8f19d3fc1768","error":"mTLS secrets not ready: certificate ray-head-cert-ttl-waiting-kjf64 not found: certificates.cert-manager.io \"ray-head-cert-ttl-waiting-kjf64\" not found"}
{"level":"info","ts":"2026-11-03T15:38:58.869Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"f1ef1e5e-14a0-4c88-927c-8f19d3fc1768","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"ttl-waiting-kjf64","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:38:58.869Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"f1ef1e5e-14a0-4c88-927c-8f19d3fc1768","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"ttl-waiting-kjf64","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:38:58.869Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"f1ef1e5e-14a0-4c88-927c-8f19d3fc1768","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"ttl-waiting-kjf64","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:38:58.869Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"f1ef1e5e-14a0-4c88-927c-8f19d3fc1768"}
{"level":"error","ts":"2026-11-03T15:38:58.869Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"f1ef1e5e-14a0-4c88-927c-8f19d3fc1768","error":"mTLS secrets not ready: certificate ray-head-cert-ttl-waiting-kjf64 not found: certificates.cert-manager.io \"ray-head-cert-ttl-waiting-kjf64\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:38:58.876Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"9dbc19ea-75ec-43a5-b926-3c4fab7428b6","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-11-03T15:38:58.877Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"9dbc19ea-75ec-43a5-b926-3c4fab7428b6","name":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:38:58.877Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"9dbc19ea-75ec-43a5-b926-3c4fab7428b6","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:38:58.877Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"9dbc19ea-75ec-43a5-b926-3c4fab7428b6"}
{"level":"info","ts":"2026-11-03T15:38:58.877Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"9dbc19ea-75ec-43a5-b926-3c4fab7428b6","RayCluster":"ttl-waiting-kjf64","State":""}
{"level":"info","ts":"2026-11-03T15:38:58.877Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"9dbc19ea-75ec-43a5-b926-3c4fab7428b6","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:38:58.930Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-waiting-kjf64","reconcileID":"1381edd1-b17e-47f2-963b-82f7fe195bea","name":"ttl-waiting-kjf64-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-11-03T15:38:58.950Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"087e6512-b39b-42c2-8f59-caee0fa2c108"}
{"level":"info","ts":"2026-11-03T15:38:58.950Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"087e6512-b39b-42c2-8f59-caee0fa2c108"}
{"level":"info","ts":"2026-11-03T15:38:58.950Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"087e6512-b39b-42c2-8f59-caee0fa2c108","1 head service found":"ttl-waiting-kjf64-head-svc"}
{"level":"info","ts":"2026-11-03T15:38:58.950Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"087e6512-b39b-42c2-8f59-caee0fa2c108"}
{"level":"info","ts":"2026-11-03T15:38:58.983Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-waiting-kjf64","reconcileID":"1381edd1-b17e-47f2-963b-82f7fe195bea","name":"kuberay-gateway-access","namespace":"test-ns-kbx4w","operation":"created"}
{"level":"info","ts":"2026-11-03T15:38:59.001Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"087e6512-b39b-42c2-8f59-caee0fa2c108","error":"certificate ray-head-cert-ttl-waiting-kjf64 is not ready at current generation 1"}
{"level":"error","ts":"2026-11-03T15:38:59.001Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"087e6512-b39b-42c2-8f59-caee0fa2c108","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-ttl-waiting-kjf64 is not ready at current generation 1","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:38:59.001Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"087e6512-b39b-42c2-8f59-caee0fa2c108","error":"mTLS secrets not ready: certificate ray-head-cert-ttl-waiting-kjf64 is not ready at current generation 1"}
{"level":"info","ts":"2026-11-03T15:38:59.001Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"087e6512-b39b-42c2-8f59-caee0fa2c108","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"ttl-waiting-kjf64","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:38:59.001Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"087e6512-b39b-42c2-8f59-caee0fa2c108","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"ttl-waiting-kjf64","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:38:59.001Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"087e6512-b39b-42c2-8f59-caee0fa2c108","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"ttl-waiting-kjf64","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:38:59.001Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"087e6512-b39b-42c2-8f59-caee0fa2c108"}
{"level":"error","ts":"2026-11-03T15:38:59.001Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"087e6512-b39b-42c2-8f59-caee0fa2c108","error":"mTLS secrets not ready: certificate ray-head-cert-ttl-waiting-kjf64 is not ready at current generation 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:38:59.037Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-waiting-kjf64","reconcileID":"1381edd1-b17e-47f2-963b-82f7fe195bea","name":"test-ns-kbx4w-ttl-waiting-kjf64","operation":"created"}
{"level":"info","ts":"2026-11-03T15:38:59.090Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-waiting-kjf64","reconcileID":"1381edd1-b17e-47f2-963b-82f7fe195bea","name":"kube-rbac-proxy-config-ttl-waiting-kjf64","operation":"created"}
{"level":"error","ts":"2026-11-03T15:38:59.144Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-waiting-kjf64","reconcileID":"1381edd1-b17e-47f2-963b-82f7fe195bea","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"ttl-waiting-kjf64\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:167\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:38:59.144Z","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-waiting-kjf64","reconcileID":"1381edd1-b17e-47f2-963b-82f7fe195bea"}
{"level":"error","ts":"2026-11-03T15:38:59.144Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-waiting-kjf64","reconcileID":"1381edd1-b17e-47f2-963b-82f7fe195bea","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"ttl-waiting-kjf64\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:38:59.144Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-waiting-kjf64","reconcileID":"d7411354-87ff-4130-907a-a9f8e1a95583","namespacedName":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:38:59.144Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-waiting-kjf64","reconcileID":"d7411354-87ff-4130-907a-a9f8e1a95583","mode":"IntegratedOAuth","cluster":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:38:59.144Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-waiting-kjf64","reconcileID":"d7411354-87ff-4130-907a-a9f8e1a95583","cluster":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:38:59.161Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"81027e98-d550-4473-9352-a3e88c869635"}
{"level":"info","ts":"2026-11-03T15:38:59.161Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"81027e98-d550-4473-9352-a3e88c869635"}
{"level":"info","ts":"2026-11-03T15:38:59.161Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"81027e98-d550-4473-9352-a3e88c869635","1 head service found":"ttl-waiting-kjf64-head-svc"}
{"level":"info","ts":"2026-11-03T15:38:59.161Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"81027e98-d550-4473-9352-a3e88c869635"}
{"level":"info","ts":"2026-11-03T15:38:59.181Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"c3d38892-343a-443f-86af-a11656818c53","certificate":"ray-head-cert-ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:38:59.181Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"c3d38892-343a-443f-86af-a11656818c53"}
{"level":"info","ts":"2026-11-03T15:38:59.181Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"8bf6c496-3be9-4d9b-84e4-96cd03aac9e2","cluster":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:38:59.205Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-waiting-kjf64","reconcileID":"d7411354-87ff-4130-907a-a9f8e1a95583","cluster":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:38:59.205Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"727630f2-4702-4802-b7b1-d3bd16e2b632","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-11-03T15:38:59.205Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"727630f2-4702-4802-b7b1-d3bd16e2b632","name":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:38:59.205Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"727630f2-4702-4802-b7b1-d3bd16e2b632","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:38:59.205Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"727630f2-4702-4802-b7b1-d3bd16e2b632"}
{"level":"info","ts":"2026-11-03T15:38:59.205Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"727630f2-4702-4802-b7b1-d3bd16e2b632","RayCluster":"ttl-waiting-kjf64","State":""}
{"level":"info","ts":"2026-11-03T15:38:59.205Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"727630f2-4702-4802-b7b1-d3bd16e2b632","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:38:59.205Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"727630f2-4702-4802-b7b1-d3bd16e2b632","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:38:59.216Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"81027e98-d550-4473-9352-a3e88c869635","error":"certificate ray-head-cert-ttl-waiting-kjf64 is not ready at current generation 1"}
{"level":"error","ts":"2026-11-03T15:38:59.216Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"81027e98-d550-4473-9352-a3e88c869635","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-ttl-waiting-kjf64 is not ready at current generation 1","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:38:59.216Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"81027e98-d550-4473-9352-a3e88c869635","error":"mTLS secrets not ready: certificate ray-head-cert-ttl-waiting-kjf64 is not ready at current generation 1"}
{"level":"info","ts":"2026-11-03T15:38:59.216Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"81027e98-d550-4473-9352-a3e88c869635","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"ttl-waiting-kjf64","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:38:59.216Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"81027e98-d550-4473-9352-a3e88c869635","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"ttl-waiting-kjf64","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:38:59.216Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"81027e98-d550-4473-9352-a3e88c869635","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"ttl-waiting-kjf64","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:38:59.216Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"81027e98-d550-4473-9352-a3e88c869635"}
{"level":"error","ts":"2026-11-03T15:38:59.216Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"81027e98-d550-4473-9352-a3e88c869635","error":"mTLS secrets not ready: certificate ray-head-cert-ttl-waiting-kjf64 is not ready at current generation 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:38:59.267Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"64080d16-3f15-491f-a786-b8f5fb206e7e","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-11-03T15:38:59.267Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"64080d16-3f15-491f-a786-b8f5fb206e7e","name":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:38:59.267Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"64080d16-3f15-491f-a786-b8f5fb206e7e","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:38:59.267Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"64080d16-3f15-491f-a786-b8f5fb206e7e"}
{"level":"info","ts":"2026-11-03T15:38:59.267Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"64080d16-3f15-491f-a786-b8f5fb206e7e","RayCluster":"ttl-waiting-kjf64","State":""}
{"level":"info","ts":"2026-11-03T15:38:59.267Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"64080d16-3f15-491f-a786-b8f5fb206e7e","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:38:59.537Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"7d2bb99b-dbcd-4c28-9855-4ac3bf003937"}
{"level":"info","ts":"2026-11-03T15:38:59.537Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"7d2bb99b-dbcd-4c28-9855-4ac3bf003937"}
{"level":"info","ts":"2026-11-03T15:38:59.537Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"7d2bb99b-dbcd-4c28-9855-4ac3bf003937","1 head service found":"ttl-waiting-kjf64-head-svc"}
{"level":"info","ts":"2026-11-03T15:38:59.537Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"7d2bb99b-dbcd-4c28-9855-4ac3bf003937"}
{"level":"info","ts":"2026-11-03T15:38:59.587Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"7d2bb99b-dbcd-4c28-9855-4ac3bf003937","error":"certificate ray-head-cert-ttl-waiting-kjf64 is not ready at current generation 1"}
{"level":"error","ts":"2026-11-03T15:38:59.587Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"7d2bb99b-dbcd-4c28-9855-4ac3bf003937","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-ttl-waiting-kjf64 is not ready at current generation 1","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:38:59.587Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"7d2bb99b-dbcd-4c28-9855-4ac3bf003937","error":"mTLS secrets not ready: certificate ray-head-cert-ttl-waiting-kjf64 is not ready at current generation 1"}
{"level":"info","ts":"2026-11-03T15:38:59.587Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"7d2bb99b-dbcd-4c28-9855-4ac3bf003937","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"ttl-waiting-kjf64","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:38:59.587Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"7d2bb99b-dbcd-4c28-9855-4ac3bf003937","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"ttl-waiting-kjf64","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:38:59.587Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"7d2bb99b-dbcd-4c28-9855-4ac3bf003937","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"ttl-waiting-kjf64","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:38:59.587Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"7d2bb99b-dbcd-4c28-9855-4ac3bf003937"}
{"level":"error","ts":"2026-11-03T15:38:59.587Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"7d2bb99b-dbcd-4c28-9855-4ac3bf003937","error":"mTLS secrets not ready: certificate ray-head-cert-ttl-waiting-kjf64 is not ready at current generation 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:38:59.635Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"8bf6c496-3be9-4d9b-84e4-96cd03aac9e2","certificate":"ray-head-cert-ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:38:59.635Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"8bf6c496-3be9-4d9b-84e4-96cd03aac9e2"}
{"level":"info","ts":"2026-11-03T15:38:59.635Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"55f9c18c-dcd1-4647-984c-6f74c3611720","cluster":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:39:00.098Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"55f9c18c-dcd1-4647-984c-6f74c3611720","certificate":"ray-head-cert-ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:39:00.098Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"55f9c18c-dcd1-4647-984c-6f74c3611720"}
{"level":"info","ts":"2026-11-03T15:39:00.227Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"c1bd53bf-157f-4009-b1db-9826bb1401b9"}
{"level":"info","ts":"2026-11-03T15:39:00.227Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"c1bd53bf-157f-4009-b1db-9826bb1401b9"}
{"level":"info","ts":"2026-11-03T15:39:00.228Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"c1bd53bf-157f-4009-b1db-9826bb1401b9","1 head service found":"ttl-waiting-kjf64-head-svc"}
{"level":"info","ts":"2026-11-03T15:39:00.228Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"c1bd53bf-157f-4009-b1db-9826bb1401b9"}
{"level":"info","ts":"2026-11-03T15:39:00.278Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"c1bd53bf-157f-4009-b1db-9826bb1401b9","error":"certificate ray-head-cert-ttl-waiting-kjf64 is not ready at current generation 1"}
{"level":"error","ts":"2026-11-03T15:39:00.278Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"c1bd53bf-157f-4009-b1db-9826bb1401b9","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-ttl-waiting-kjf64 is not ready at current generation 1","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:39:00.278Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"c1bd53bf-157f-4009-b1db-9826bb1401b9","error":"mTLS secrets not ready: certificate ray-head-cert-ttl-waiting-kjf64 is not ready at current generation 1"}
{"level":"info","ts":"2026-11-03T15:39:00.278Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"c1bd53bf-157f-4009-b1db-9826bb1401b9","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"ttl-waiting-kjf64","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:39:00.278Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"c1bd53bf-157f-4009-b1db-9826bb1401b9","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"ttl-waiting-kjf64","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:39:00.278Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"c1bd53bf-157f-4009-b1db-9826bb1401b9","filter labels":["test-ns-kbx4w",{"ray.io/cluster":"ttl-waiting-kjf64","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:39:00.278Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"c1bd53bf-157f-4009-b1db-9826bb1401b9"}
{"level":"error","ts":"2026-11-03T15:39:00.278Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"c1bd53bf-157f-4009-b1db-9826bb1401b9","error":"mTLS secrets not ready: certificate ray-head-cert-ttl-waiting-kjf64 is not ready at current generation 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:39:01.559Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"97021ff2-6a2c-4adc-908a-e8673bd1453b"}
{"level":"info","ts":"2026-11-03T15:39:01.559Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"97021ff2-6a2c-4adc-908a-e8673bd1453b"}
{"level":"info","ts":"2026-11-03T15:39:01.559Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"97021ff2-6a2c-4adc-908a-e8673bd1453b","1 head service found":"ttl-waiting-kjf64-head-svc"}
{"level":"info","ts":"2026-11-03T15:39:01.559Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"97021ff2-6a2c-4adc-908a-e8673bd1453b"}
{"level":"info","ts":"2026-11-03T15:39:01.661Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"97021ff2-6a2c-4adc-908a-e8673bd1453b","mode":"IntegratedOAuth","cluster":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:39:01.661Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"97021ff2-6a2c-4adc-908a-e8673bd1453b","cluster":"ttl-waiting-kjf64","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-11-03T15:39:01.661Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"97021ff2-6a2c-4adc-908a-e8673bd1453b","cluster":"ttl-waiting-kjf64","authType":"IntegratedOAuth","serviceAccount":"ttl-waiting-kjf64-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-11-03T15:39:01.661Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"97021ff2-6a2c-4adc-908a-e8673bd1453b","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"ttl-waiting-kjf64","ray.io/group":"headgroup","ray.io/identifier":"ttl-waiting-kjf64-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-11-03T15:39:01.661Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"97021ff2-6a2c-4adc-908a-e8673bd1453b","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-11-03T15:39:01.661Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"97021ff2-6a2c-4adc-908a-e8673bd1453b","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-11-03T15:39:01.661Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"97021ff2-6a2c-4adc-908a-e8673bd1453b","rayNodeType":"head","generatedCmd":"ulimit -n ${RAY_START_ULIMIT_OPEN_FILES:-65536}; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-11-03T15:39:01.661Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"97021ff2-6a2c-4adc-908a-e8673bd1453b","enabled":true}
{"level":"info","ts":"2026-11-03T15:39:01.773Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"97021ff2-6a2c-4adc-908a-e8673bd1453b","name":"ttl-waiting-kjf64-head-dvbng"}
{"level":"info","ts":"2026-11-03T15:39:01.773Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"97021ff2-6a2c-4adc-908a-e8673bd1453b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:39:01.773Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"97021ff2-6a2c-4adc-908a-e8673bd1453b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:39:01.773Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"97021ff2-6a2c-4adc-908a-e8673bd1453b","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-11-03T15:39:01.773Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"97021ff2-6a2c-4adc-908a-e8673bd1453b","group":"small-group","indices":{}}
{"level":"info","ts":"2026-11-03T15:39:01.773Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"97021ff2-6a2c-4adc-908a-e8673bd1453b","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-11-03T15:39:01.874Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"97021ff2-6a2c-4adc-908a-e8673bd1453b","creating worker for group":"small-group","index":0,"total":1,"replicaIndex":0}
{"level":"info","ts":"2026-11-03T15:39:01.874Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"97021ff2-6a2c-4adc-908a-e8673bd1453b","nodeType":"worker","rayStartParams":{"address":"ttl-waiting-kjf64-head-svc.test-ns-kbx4w.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-11-03T15:39:01.874Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"97021ff2-6a2c-4adc-908a-e8673bd1453b","rayStartCmd":"ray start  --address=ttl-waiting-kjf64-head-svc.test-ns-kbx4w.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-11-03T15:39:01.874Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"97021ff2-6a2c-4adc-908a-e8673bd1453b","rayNodeType":"worker","generatedCmd":"ulimit -n ${RAY_START_ULIMIT_OPEN_FILES:-65536}; ray start  --address=ttl-waiting-kjf64-head-svc.test-ns-kbx4w.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-11-03T15:39:01.874Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"97021ff2-6a2c-4adc-908a-e8673bd1453b","enabled":true}
{"level":"info","ts":"2026-11-03T15:39:01.936Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"97021ff2-6a2c-4adc-908a-e8673bd1453b","name":"ttl-waiting-kjf64-small-group-worker-blcp8"}
{"level":"info","ts":"2026-11-03T15:39:01.937Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"97021ff2-6a2c-4adc-908a-e8673bd1453b","podName":"ttl-waiting-kjf64-head-dvbng","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:39:01.937Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"97021ff2-6a2c-4adc-908a-e8673bd1453b","podName":"ttl-waiting-kjf64-head-dvbng","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:39:01.937Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"97021ff2-6a2c-4adc-908a-e8673bd1453b","name":"ttl-waiting-kjf64","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:39:01Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:39:01.998Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"51a1490a-2958-4a21-9002-621252e05f9b"}
{"level":"info","ts":"2026-11-03T15:39:01.998Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"51a1490a-2958-4a21-9002-621252e05f9b"}
{"level":"info","ts":"2026-11-03T15:39:01.998Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"51a1490a-2958-4a21-9002-621252e05f9b","1 head service found":"ttl-waiting-kjf64-head-svc"}
{"level":"info","ts":"2026-11-03T15:39:01.998Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"51a1490a-2958-4a21-9002-621252e05f9b","Found 1 head Pod":"ttl-waiting-kjf64-head-dvbng","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:39:01.998Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"51a1490a-2958-4a21-9002-621252e05f9b","head Pod":"ttl-waiting-kjf64-head-dvbng","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ttl-waiting-kjf64-head-dvbng. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:39:01.998Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"51a1490a-2958-4a21-9002-621252e05f9b","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:39:01.999Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"51a1490a-2958-4a21-9002-621252e05f9b","worker Pod":"ttl-waiting-kjf64-small-group-worker-blcp8","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ttl-waiting-kjf64-small-group-worker-blcp8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:39:01.999Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"51a1490a-2958-4a21-9002-621252e05f9b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:39:01.999Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"51a1490a-2958-4a21-9002-621252e05f9b","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:39:01.999Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"51a1490a-2958-4a21-9002-621252e05f9b","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:39:01.999Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"51a1490a-2958-4a21-9002-621252e05f9b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:39:01.999Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"51a1490a-2958-4a21-9002-621252e05f9b","podName":"ttl-waiting-kjf64-small-group-worker-blcp8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:39:01.999Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"51a1490a-2958-4a21-9002-621252e05f9b","podName":"ttl-waiting-kjf64-small-group-worker-blcp8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:39:01.999Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"51a1490a-2958-4a21-9002-621252e05f9b","name":"ttl-waiting-kjf64","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:39:01Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:39:02.001Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"a1ac756e-a9d0-4f63-be11-0a8e964c84e3","cluster":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:39:02.001Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"32e0a838-7d23-4150-82a9-50158f350b0f","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-11-03T15:39:02.001Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"32e0a838-7d23-4150-82a9-50158f350b0f","name":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:39:02.001Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"32e0a838-7d23-4150-82a9-50158f350b0f","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:39:02.001Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"32e0a838-7d23-4150-82a9-50158f350b0f"}
{"level":"info","ts":"2026-11-03T15:39:02.001Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"32e0a838-7d23-4150-82a9-50158f350b0f","RayCluster":"ttl-waiting-kjf64","State":""}
{"level":"info","ts":"2026-11-03T15:39:02.001Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"32e0a838-7d23-4150-82a9-50158f350b0f","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:38:58Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:39:01Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:39:02.001Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"32e0a838-7d23-4150-82a9-50158f350b0f","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:39:02.054Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"51a1490a-2958-4a21-9002-621252e05f9b","name":"ttl-waiting-kjf64","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ttl-waiting-kjf64\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-kbx4w","name":"ttl-waiting-kjf64"}}
{"level":"info","ts":"2026-11-03T15:39:02.054Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"51a1490a-2958-4a21-9002-621252e05f9b"}
{"level":"error","ts":"2026-11-03T15:39:02.054Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"51a1490a-2958-4a21-9002-621252e05f9b","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ttl-waiting-kjf64\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:39:02.054Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"670d8ba8-873e-4a98-83a6-4748e71cc84d"}
{"level":"info","ts":"2026-11-03T15:39:02.054Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"670d8ba8-873e-4a98-83a6-4748e71cc84d"}
{"level":"info","ts":"2026-11-03T15:39:02.054Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"670d8ba8-873e-4a98-83a6-4748e71cc84d","1 head service found":"ttl-waiting-kjf64-head-svc"}
{"level":"info","ts":"2026-11-03T15:39:02.054Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"670d8ba8-873e-4a98-83a6-4748e71cc84d","Found 1 head Pod":"ttl-waiting-kjf64-head-dvbng","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:39:02.054Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"670d8ba8-873e-4a98-83a6-4748e71cc84d","head Pod":"ttl-waiting-kjf64-head-dvbng","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ttl-waiting-kjf64-head-dvbng. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:39:02.054Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"670d8ba8-873e-4a98-83a6-4748e71cc84d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:39:02.054Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"670d8ba8-873e-4a98-83a6-4748e71cc84d","worker Pod":"ttl-waiting-kjf64-small-group-worker-blcp8","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ttl-waiting-kjf64-small-group-worker-blcp8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:39:02.054Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"670d8ba8-873e-4a98-83a6-4748e71cc84d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:39:02.054Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"670d8ba8-873e-4a98-83a6-4748e71cc84d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:39:02.054Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"670d8ba8-873e-4a98-83a6-4748e71cc84d","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:39:02.054Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"670d8ba8-873e-4a98-83a6-4748e71cc84d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:39:02.054Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"670d8ba8-873e-4a98-83a6-4748e71cc84d","podName":"ttl-waiting-kjf64-head-dvbng","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:39:02.054Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"670d8ba8-873e-4a98-83a6-4748e71cc84d","podName":"ttl-waiting-kjf64-head-dvbng","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:39:02.055Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"670d8ba8-873e-4a98-83a6-4748e71cc84d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:39:02.055Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"670d8ba8-873e-4a98-83a6-4748e71cc84d","seconds":300}
{"level":"info","ts":"2026-11-03T15:39:02.063Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"22bcbb07-9657-4421-a39e-a85ee6257a29","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-11-03T15:39:02.063Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"22bcbb07-9657-4421-a39e-a85ee6257a29","name":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:39:02.063Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"22bcbb07-9657-4421-a39e-a85ee6257a29","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:39:02.063Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"22bcbb07-9657-4421-a39e-a85ee6257a29"}
{"level":"info","ts":"2026-11-03T15:39:02.063Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"22bcbb07-9657-4421-a39e-a85ee6257a29","RayCluster":"ttl-waiting-kjf64","State":""}
{"level":"info","ts":"2026-11-03T15:39:02.063Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"22bcbb07-9657-4421-a39e-a85ee6257a29","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:39:01Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:39:01Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:39:02.069Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"5dcb130b-c426-43ea-a257-597de8b3618c"}
{"level":"info","ts":"2026-11-03T15:39:02.069Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"5dcb130b-c426-43ea-a257-597de8b3618c"}
{"level":"info","ts":"2026-11-03T15:39:02.070Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"5dcb130b-c426-43ea-a257-597de8b3618c","1 head service found":"ttl-waiting-kjf64-head-svc"}
{"level":"info","ts":"2026-11-03T15:39:02.070Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"5dcb130b-c426-43ea-a257-597de8b3618c","Found 1 head Pod":"ttl-waiting-kjf64-head-dvbng","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:39:02.070Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"5dcb130b-c426-43ea-a257-597de8b3618c","head Pod":"ttl-waiting-kjf64-head-dvbng","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ttl-waiting-kjf64-head-dvbng. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:39:02.070Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"5dcb130b-c426-43ea-a257-597de8b3618c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:39:02.070Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"5dcb130b-c426-43ea-a257-597de8b3618c","worker Pod":"ttl-waiting-kjf64-small-group-worker-blcp8","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ttl-waiting-kjf64-small-group-worker-blcp8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:39:02.070Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"5dcb130b-c426-43ea-a257-597de8b3618c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:39:02.070Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"5dcb130b-c426-43ea-a257-597de8b3618c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:39:02.070Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"5dcb130b-c426-43ea-a257-597de8b3618c","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:39:02.070Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"5dcb130b-c426-43ea-a257-597de8b3618c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:39:02.070Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"5dcb130b-c426-43ea-a257-597de8b3618c","podName":"ttl-waiting-kjf64-head-dvbng","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:39:02.070Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"5dcb130b-c426-43ea-a257-597de8b3618c","podName":"ttl-waiting-kjf64-head-dvbng","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:39:02.070Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"5dcb130b-c426-43ea-a257-597de8b3618c","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:39:02.070Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"5dcb130b-c426-43ea-a257-597de8b3618c","seconds":300}
{"level":"info","ts":"2026-11-03T15:39:02.483Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"93905838-ad7d-474d-b112-01bc9bf24777"}
{"level":"info","ts":"2026-11-03T15:39:02.483Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"93905838-ad7d-474d-b112-01bc9bf24777"}
{"level":"info","ts":"2026-11-03T15:39:02.483Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"93905838-ad7d-474d-b112-01bc9bf24777","1 head service found":"ttl-waiting-kjf64-head-svc"}
{"level":"info","ts":"2026-11-03T15:39:02.483Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"93905838-ad7d-474d-b112-01bc9bf24777","Found 1 head Pod":"ttl-waiting-kjf64-head-dvbng","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:39:02.483Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"93905838-ad7d-474d-b112-01bc9bf24777","head Pod":"ttl-waiting-kjf64-head-dvbng","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ttl-waiting-kjf64-head-dvbng. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:39:02.483Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"93905838-ad7d-474d-b112-01bc9bf24777","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:39:02.483Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"93905838-ad7d-474d-b112-01bc9bf24777","worker Pod":"ttl-waiting-kjf64-small-group-worker-blcp8","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ttl-waiting-kjf64-small-group-worker-blcp8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:39:02.483Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"93905838-ad7d-474d-b112-01bc9bf24777","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:39:02.483Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"93905838-ad7d-474d-b112-01bc9bf24777","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:39:02.483Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"93905838-ad7d-474d-b112-01bc9bf24777","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:39:02.483Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"93905838-ad7d-474d-b112-01bc9bf24777","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:39:02.483Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"93905838-ad7d-474d-b112-01bc9bf24777","podName":"ttl-waiting-kjf64-head-dvbng","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:39:02.483Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"93905838-ad7d-474d-b112-01bc9bf24777","podName":"ttl-waiting-kjf64-head-dvbng","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:39:02.483Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"93905838-ad7d-474d-b112-01bc9bf24777","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:39:02.483Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"93905838-ad7d-474d-b112-01bc9bf24777","seconds":300}
{"level":"info","ts":"2026-11-03T15:39:02.743Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"795d3f7f-1033-45a0-a63b-a87e558cc241"}
{"level":"info","ts":"2026-11-03T15:39:02.743Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"795d3f7f-1033-45a0-a63b-a87e558cc241"}
{"level":"info","ts":"2026-11-03T15:39:02.744Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"795d3f7f-1033-45a0-a63b-a87e558cc241","1 head service found":"ttl-waiting-kjf64-head-svc"}
{"level":"info","ts":"2026-11-03T15:39:02.744Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"795d3f7f-1033-45a0-a63b-a87e558cc241","Found 1 head Pod":"ttl-waiting-kjf64-head-dvbng","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:39:02.744Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"795d3f7f-1033-45a0-a63b-a87e558cc241","head Pod":"ttl-waiting-kjf64-head-dvbng","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ttl-waiting-kjf64-head-dvbng. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:39:02.744Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"795d3f7f-1033-45a0-a63b-a87e558cc241","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:39:02.744Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"795d3f7f-1033-45a0-a63b-a87e558cc241","worker Pod":"ttl-waiting-kjf64-small-group-worker-blcp8","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ttl-waiting-kjf64-small-group-worker-blcp8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:39:02.744Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"795d3f7f-1033-45a0-a63b-a87e558cc241","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:39:02.744Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"795d3f7f-1033-45a0-a63b-a87e558cc241","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:39:02.744Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"795d3f7f-1033-45a0-a63b-a87e558cc241","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:39:02.744Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"795d3f7f-1033-45a0-a63b-a87e558cc241","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:39:02.744Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"795d3f7f-1033-45a0-a63b-a87e558cc241","podName":"ttl-waiting-kjf64-head-dvbng","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:39:02.744Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"795d3f7f-1033-45a0-a63b-a87e558cc241","podName":"ttl-waiting-kjf64-head-dvbng","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:39:02.744Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"795d3f7f-1033-45a0-a63b-a87e558cc241","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:39:02.744Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"795d3f7f-1033-45a0-a63b-a87e558cc241","seconds":300}
{"level":"info","ts":"2026-11-03T15:39:02.823Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"692f4176-8c3b-4074-9930-7a32c52e8b8f"}
{"level":"info","ts":"2026-11-03T15:39:02.823Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"692f4176-8c3b-4074-9930-7a32c52e8b8f"}
{"level":"info","ts":"2026-11-03T15:39:02.823Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"692f4176-8c3b-4074-9930-7a32c52e8b8f","1 head service found":"ttl-waiting-kjf64-head-svc"}
{"level":"info","ts":"2026-11-03T15:39:02.823Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"692f4176-8c3b-4074-9930-7a32c52e8b8f","Found 1 head Pod":"ttl-waiting-kjf64-head-dvbng","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:39:02.823Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"692f4176-8c3b-4074-9930-7a32c52e8b8f","head Pod":"ttl-waiting-kjf64-head-dvbng","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ttl-waiting-kjf64-head-dvbng. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:39:02.823Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"692f4176-8c3b-4074-9930-7a32c52e8b8f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:39:02.823Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"692f4176-8c3b-4074-9930-7a32c52e8b8f","worker Pod":"ttl-waiting-kjf64-small-group-worker-blcp8","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ttl-waiting-kjf64-small-group-worker-blcp8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:39:02.823Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"692f4176-8c3b-4074-9930-7a32c52e8b8f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:39:02.823Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"692f4176-8c3b-4074-9930-7a32c52e8b8f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:39:02.823Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"692f4176-8c3b-4074-9930-7a32c52e8b8f","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:39:02.823Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"692f4176-8c3b-4074-9930-7a32c52e8b8f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:39:02.823Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"692f4176-8c3b-4074-9930-7a32c52e8b8f","podName":"ttl-waiting-kjf64-small-group-worker-blcp8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:39:02.823Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"692f4176-8c3b-4074-9930-7a32c52e8b8f","podName":"ttl-waiting-kjf64-small-group-worker-blcp8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:39:02.824Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"692f4176-8c3b-4074-9930-7a32c52e8b8f","name":"ttl-waiting-kjf64","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:39:01Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:39:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:39:02.884Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"9bc6fbdf-9097-4283-bae3-2d0ec17156f9","cluster":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:39:02.884Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"cfdee74e-7695-482d-8702-6a1031fac030","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-11-03T15:39:02.884Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"cfdee74e-7695-482d-8702-6a1031fac030","name":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:39:02.884Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"cfdee74e-7695-482d-8702-6a1031fac030","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:39:02.885Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"cfdee74e-7695-482d-8702-6a1031fac030"}
{"level":"info","ts":"2026-11-03T15:39:02.885Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"cfdee74e-7695-482d-8702-6a1031fac030","RayCluster":"ttl-waiting-kjf64","State":""}
{"level":"info","ts":"2026-11-03T15:39:02.885Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"cfdee74e-7695-482d-8702-6a1031fac030","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:39:01Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:39:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:39:02.885Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"cfdee74e-7695-482d-8702-6a1031fac030","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:39:02.944Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"4d979d6b-3d02-4b56-933d-6deec754bca4","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-11-03T15:39:02.944Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"4d979d6b-3d02-4b56-933d-6deec754bca4","name":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:39:02.944Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"4d979d6b-3d02-4b56-933d-6deec754bca4","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:39:02.945Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"4d979d6b-3d02-4b56-933d-6deec754bca4"}
{"level":"info","ts":"2026-11-03T15:39:02.945Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"4d979d6b-3d02-4b56-933d-6deec754bca4","RayCluster":"ttl-waiting-kjf64","State":""}
{"level":"info","ts":"2026-11-03T15:39:02.945Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"4d979d6b-3d02-4b56-933d-6deec754bca4","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:39:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:39:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:39:03.091Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"f9af1354-c075-4a79-aa64-91ed46986050"}
{"level":"info","ts":"2026-11-03T15:39:03.091Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"f9af1354-c075-4a79-aa64-91ed46986050"}
{"level":"info","ts":"2026-11-03T15:39:03.091Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"f9af1354-c075-4a79-aa64-91ed46986050","1 head service found":"ttl-waiting-kjf64-head-svc"}
{"level":"info","ts":"2026-11-03T15:39:03.091Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"f9af1354-c075-4a79-aa64-91ed46986050","Found 1 head Pod":"ttl-waiting-kjf64-head-dvbng","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:39:03.091Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"f9af1354-c075-4a79-aa64-91ed46986050","head Pod":"ttl-waiting-kjf64-head-dvbng","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ttl-waiting-kjf64-head-dvbng. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:39:03.091Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"f9af1354-c075-4a79-aa64-91ed46986050","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:39:03.091Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"f9af1354-c075-4a79-aa64-91ed46986050","worker Pod":"ttl-waiting-kjf64-small-group-worker-blcp8","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ttl-waiting-kjf64-small-group-worker-blcp8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:39:03.091Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"f9af1354-c075-4a79-aa64-91ed46986050","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:39:03.091Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"f9af1354-c075-4a79-aa64-91ed46986050","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:39:03.091Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"f9af1354-c075-4a79-aa64-91ed46986050","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:39:03.091Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"f9af1354-c075-4a79-aa64-91ed46986050","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:39:03.091Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"f9af1354-c075-4a79-aa64-91ed46986050","podName":"ttl-waiting-kjf64-small-group-worker-blcp8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:39:03.091Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"f9af1354-c075-4a79-aa64-91ed46986050","podName":"ttl-waiting-kjf64-small-group-worker-blcp8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:39:03.091Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"f9af1354-c075-4a79-aa64-91ed46986050","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:39:03.091Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"f9af1354-c075-4a79-aa64-91ed46986050","seconds":300}
{"level":"info","ts":"2026-11-03T15:39:03.186Z","logger":"controllers.RayClusterMTLS","msg":"Updating head certificate","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"9bc6fbdf-9097-4283-bae3-2d0ec17156f9","certificate":"ray-head-cert-ttl-waiting-kjf64","ipAddresses":["10.132.0.51","127.0.0.1"]}
{"level":"info","ts":"2026-11-03T15:39:03.445Z","logger":"controllers.RayClusterMTLS","msg":"Updating worker certificate","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"9bc6fbdf-9097-4283-bae3-2d0ec17156f9","certificate":"ray-worker-cert-ttl-waiting-kjf64","ipAddresses":["10.134.0.55","127.0.0.1"]}
{"level":"info","ts":"2026-11-03T15:39:03.705Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"9bc6fbdf-9097-4283-bae3-2d0ec17156f9","certificate":"ray-head-cert-ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:39:03.705Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"9bc6fbdf-9097-4283-bae3-2d0ec17156f9"}
{"level":"info","ts":"2026-11-03T15:39:05.946Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"19c3b1fd-b59e-40fd-96f5-65001727fac3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-11-03T15:39:05.946Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"19c3b1fd-b59e-40fd-96f5-65001727fac3","name":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:39:05.946Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"19c3b1fd-b59e-40fd-96f5-65001727fac3","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:39:05.946Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"19c3b1fd-b59e-40fd-96f5-65001727fac3"}
{"level":"info","ts":"2026-11-03T15:39:05.946Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"19c3b1fd-b59e-40fd-96f5-65001727fac3","RayCluster":"ttl-waiting-kjf64","State":""}
{"level":"info","ts":"2026-11-03T15:39:05.946Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"19c3b1fd-b59e-40fd-96f5-65001727fac3","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:39:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:39:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:39:08.947Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"f646a16d-7afa-49fd-a719-5132b50233f1","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-11-03T15:39:08.947Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"f646a16d-7afa-49fd-a719-5132b50233f1","name":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:39:08.947Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"f646a16d-7afa-49fd-a719-5132b50233f1","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:39:08.947Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"f646a16d-7afa-49fd-a719-5132b50233f1"}
{"level":"info","ts":"2026-11-03T15:39:08.947Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"f646a16d-7afa-49fd-a719-5132b50233f1","RayCluster":"ttl-waiting-kjf64","State":""}
{"level":"info","ts":"2026-11-03T15:39:08.947Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"f646a16d-7afa-49fd-a719-5132b50233f1","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:39:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:39:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:39:11.947Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"e52f712f-1eff-4be2-86ef-8c0c53e0dd77","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-11-03T15:39:11.947Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"e52f712f-1eff-4be2-86ef-8c0c53e0dd77","name":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:39:11.947Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"e52f712f-1eff-4be2-86ef-8c0c53e0dd77","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:39:11.947Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"e52f712f-1eff-4be2-86ef-8c0c53e0dd77"}
{"level":"info","ts":"2026-11-03T15:39:11.947Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"e52f712f-1eff-4be2-86ef-8c0c53e0dd77","RayCluster":"ttl-waiting-kjf64","State":""}
{"level":"info","ts":"2026-11-03T15:39:11.947Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"e52f712f-1eff-4be2-86ef-8c0c53e0dd77","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:39:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:39:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:39:14.948Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"5488631d-5144-4420-8c58-e144d41de7a4","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-11-03T15:39:14.948Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"5488631d-5144-4420-8c58-e144d41de7a4","name":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:39:14.948Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"5488631d-5144-4420-8c58-e144d41de7a4","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:39:14.949Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"5488631d-5144-4420-8c58-e144d41de7a4"}
{"level":"info","ts":"2026-11-03T15:39:14.949Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"5488631d-5144-4420-8c58-e144d41de7a4","RayCluster":"ttl-waiting-kjf64","State":""}
{"level":"info","ts":"2026-11-03T15:39:14.949Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"5488631d-5144-4420-8c58-e144d41de7a4","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:39:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:39:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:39:17.949Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"2c97cc3d-c0fc-48a8-9f05-e24de92b3af8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-11-03T15:39:17.949Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"2c97cc3d-c0fc-48a8-9f05-e24de92b3af8","name":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:39:17.949Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"2c97cc3d-c0fc-48a8-9f05-e24de92b3af8","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:39:17.949Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"2c97cc3d-c0fc-48a8-9f05-e24de92b3af8"}
{"level":"info","ts":"2026-11-03T15:39:17.949Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"2c97cc3d-c0fc-48a8-9f05-e24de92b3af8","RayCluster":"ttl-waiting-kjf64","State":""}
{"level":"info","ts":"2026-11-03T15:39:17.949Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"2c97cc3d-c0fc-48a8-9f05-e24de92b3af8","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:39:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:39:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:39:20.950Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"017b8fa0-2631-4d85-bcf8-cce0021db98c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-11-03T15:39:20.950Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"017b8fa0-2631-4d85-bcf8-cce0021db98c","name":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:39:20.950Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"017b8fa0-2631-4d85-bcf8-cce0021db98c","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:39:20.951Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"017b8fa0-2631-4d85-bcf8-cce0021db98c"}
{"level":"info","ts":"2026-11-03T15:39:20.951Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"017b8fa0-2631-4d85-bcf8-cce0021db98c","RayCluster":"ttl-waiting-kjf64","State":""}
{"level":"info","ts":"2026-11-03T15:39:20.951Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"017b8fa0-2631-4d85-bcf8-cce0021db98c","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:39:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:39:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:39:23.951Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"424ff956-bbff-4f60-b3dd-cb2016ab800b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-11-03T15:39:23.951Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"424ff956-bbff-4f60-b3dd-cb2016ab800b","name":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:39:23.951Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"424ff956-bbff-4f60-b3dd-cb2016ab800b","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:39:23.951Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"424ff956-bbff-4f60-b3dd-cb2016ab800b"}
{"level":"info","ts":"2026-11-03T15:39:23.951Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"424ff956-bbff-4f60-b3dd-cb2016ab800b","RayCluster":"ttl-waiting-kjf64","State":""}
{"level":"info","ts":"2026-11-03T15:39:23.951Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"424ff956-bbff-4f60-b3dd-cb2016ab800b","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:39:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:39:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:39:26.952Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"78c59ff3-5907-4adf-96d2-a6e204b98aba","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-11-03T15:39:26.952Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"78c59ff3-5907-4adf-96d2-a6e204b98aba","name":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:39:26.952Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"78c59ff3-5907-4adf-96d2-a6e204b98aba","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:39:26.953Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"78c59ff3-5907-4adf-96d2-a6e204b98aba"}
{"level":"info","ts":"2026-11-03T15:39:26.953Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"78c59ff3-5907-4adf-96d2-a6e204b98aba","RayCluster":"ttl-waiting-kjf64","State":""}
{"level":"info","ts":"2026-11-03T15:39:26.953Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"78c59ff3-5907-4adf-96d2-a6e204b98aba","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:39:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:39:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:39:29.953Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"a04bfc4c-fb51-4698-94a7-c2a3a520f972","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-11-03T15:39:29.953Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"a04bfc4c-fb51-4698-94a7-c2a3a520f972","name":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:39:29.953Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"a04bfc4c-fb51-4698-94a7-c2a3a520f972","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:39:29.954Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"a04bfc4c-fb51-4698-94a7-c2a3a520f972"}
{"level":"info","ts":"2026-11-03T15:39:29.954Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"a04bfc4c-fb51-4698-94a7-c2a3a520f972","RayCluster":"ttl-waiting-kjf64","State":""}
{"level":"info","ts":"2026-11-03T15:39:29.954Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"a04bfc4c-fb51-4698-94a7-c2a3a520f972","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:39:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:39:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:39:32.955Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"e7d202d0-a05c-4fef-933c-401c11940c47","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-11-03T15:39:32.955Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"e7d202d0-a05c-4fef-933c-401c11940c47","name":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:39:32.955Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"e7d202d0-a05c-4fef-933c-401c11940c47","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:39:32.955Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"e7d202d0-a05c-4fef-933c-401c11940c47"}
{"level":"info","ts":"2026-11-03T15:39:32.955Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"e7d202d0-a05c-4fef-933c-401c11940c47","RayCluster":"ttl-waiting-kjf64","State":""}
{"level":"info","ts":"2026-11-03T15:39:32.955Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"e7d202d0-a05c-4fef-933c-401c11940c47","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:39:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:39:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:39:33.705Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"15696b4c-990b-45f9-9c19-2a097a0efdc9","cluster":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:39:35.955Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"64d96c8f-9404-4183-8e1a-3280ec818237","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-11-03T15:39:35.955Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"64d96c8f-9404-4183-8e1a-3280ec818237","name":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:39:35.955Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"64d96c8f-9404-4183-8e1a-3280ec818237","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:39:35.956Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"64d96c8f-9404-4183-8e1a-3280ec818237"}
{"level":"info","ts":"2026-11-03T15:39:35.956Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"64d96c8f-9404-4183-8e1a-3280ec818237","RayCluster":"ttl-waiting-kjf64","State":""}
{"level":"info","ts":"2026-11-03T15:39:35.956Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"64d96c8f-9404-4183-8e1a-3280ec818237","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:39:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:39:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:39:38.956Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"d6226872-4b02-4a65-8cb3-0715461df1e8","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-11-03T15:39:38.956Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"d6226872-4b02-4a65-8cb3-0715461df1e8","name":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:39:38.956Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"d6226872-4b02-4a65-8cb3-0715461df1e8","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:39:38.956Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"d6226872-4b02-4a65-8cb3-0715461df1e8"}
{"level":"info","ts":"2026-11-03T15:39:38.956Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"d6226872-4b02-4a65-8cb3-0715461df1e8","RayCluster":"ttl-waiting-kjf64","State":""}
{"level":"info","ts":"2026-11-03T15:39:38.956Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"d6226872-4b02-4a65-8cb3-0715461df1e8","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:39:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:39:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:39:41.957Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"915ad2bc-ccfe-44e6-ad0b-a0ca6d77a51d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-11-03T15:39:41.957Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"915ad2bc-ccfe-44e6-ad0b-a0ca6d77a51d","name":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:39:41.957Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"915ad2bc-ccfe-44e6-ad0b-a0ca6d77a51d","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:39:41.957Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"915ad2bc-ccfe-44e6-ad0b-a0ca6d77a51d"}
{"level":"info","ts":"2026-11-03T15:39:41.957Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"915ad2bc-ccfe-44e6-ad0b-a0ca6d77a51d","RayCluster":"ttl-waiting-kjf64","State":""}
{"level":"info","ts":"2026-11-03T15:39:41.957Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"915ad2bc-ccfe-44e6-ad0b-a0ca6d77a51d","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:39:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:39:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:39:44.958Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"28621b10-a388-4151-bf80-d511860e3e46","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-11-03T15:39:44.958Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"28621b10-a388-4151-bf80-d511860e3e46","name":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:39:44.958Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"28621b10-a388-4151-bf80-d511860e3e46","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:39:44.958Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"28621b10-a388-4151-bf80-d511860e3e46"}
{"level":"info","ts":"2026-11-03T15:39:44.958Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"28621b10-a388-4151-bf80-d511860e3e46","RayCluster":"ttl-waiting-kjf64","State":""}
{"level":"info","ts":"2026-11-03T15:39:44.958Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"28621b10-a388-4151-bf80-d511860e3e46","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:39:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:39:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:39:47.959Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"3568664c-4a95-40e5-b19a-87d4629089e2","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-11-03T15:39:47.959Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"3568664c-4a95-40e5-b19a-87d4629089e2","name":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:39:47.959Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"3568664c-4a95-40e5-b19a-87d4629089e2","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:39:47.959Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"3568664c-4a95-40e5-b19a-87d4629089e2"}
{"level":"info","ts":"2026-11-03T15:39:47.959Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"3568664c-4a95-40e5-b19a-87d4629089e2","RayCluster":"ttl-waiting-kjf64","State":""}
{"level":"info","ts":"2026-11-03T15:39:47.959Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"3568664c-4a95-40e5-b19a-87d4629089e2","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:39:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:39:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:39:50.960Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"80d577bc-1b65-4df1-814d-4f372195bf4c","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-11-03T15:39:50.960Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"80d577bc-1b65-4df1-814d-4f372195bf4c","name":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:39:50.960Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"80d577bc-1b65-4df1-814d-4f372195bf4c","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:39:50.960Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"80d577bc-1b65-4df1-814d-4f372195bf4c"}
{"level":"info","ts":"2026-11-03T15:39:50.960Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"80d577bc-1b65-4df1-814d-4f372195bf4c","RayCluster":"ttl-waiting-kjf64","State":""}
{"level":"info","ts":"2026-11-03T15:39:50.960Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"80d577bc-1b65-4df1-814d-4f372195bf4c","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:39:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:39:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:39:53.960Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"881a4ff3-1765-4904-82c7-09a1bb91dc1a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-11-03T15:39:53.960Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"881a4ff3-1765-4904-82c7-09a1bb91dc1a","name":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:39:53.960Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"881a4ff3-1765-4904-82c7-09a1bb91dc1a","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:39:53.960Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"881a4ff3-1765-4904-82c7-09a1bb91dc1a"}
{"level":"info","ts":"2026-11-03T15:39:53.960Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"881a4ff3-1765-4904-82c7-09a1bb91dc1a","RayCluster":"ttl-waiting-kjf64","State":""}
{"level":"info","ts":"2026-11-03T15:39:53.960Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"881a4ff3-1765-4904-82c7-09a1bb91dc1a","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:39:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:39:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:39:56.961Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"a59a1082-2a8f-472e-9625-b84cf51cd066","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-11-03T15:39:56.961Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"a59a1082-2a8f-472e-9625-b84cf51cd066","name":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:39:56.961Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"a59a1082-2a8f-472e-9625-b84cf51cd066","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:39:56.961Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"a59a1082-2a8f-472e-9625-b84cf51cd066"}
{"level":"info","ts":"2026-11-03T15:39:56.961Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"a59a1082-2a8f-472e-9625-b84cf51cd066","RayCluster":"ttl-waiting-kjf64","State":""}
{"level":"info","ts":"2026-11-03T15:39:56.961Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"a59a1082-2a8f-472e-9625-b84cf51cd066","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:39:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:39:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:39:59.962Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"e76a2bc5-e626-465f-90df-85462f81e706","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-11-03T15:39:59.962Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"e76a2bc5-e626-465f-90df-85462f81e706","name":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:39:59.962Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"e76a2bc5-e626-465f-90df-85462f81e706","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:39:59.962Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"e76a2bc5-e626-465f-90df-85462f81e706"}
{"level":"info","ts":"2026-11-03T15:39:59.962Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"e76a2bc5-e626-465f-90df-85462f81e706","RayCluster":"ttl-waiting-kjf64","State":""}
{"level":"info","ts":"2026-11-03T15:39:59.962Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"e76a2bc5-e626-465f-90df-85462f81e706","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:39:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:39:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:40:02.964Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"79bfa9c9-51a1-475b-b821-c9d144ab8d2a","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-11-03T15:40:02.964Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"79bfa9c9-51a1-475b-b821-c9d144ab8d2a","name":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:40:02.964Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"79bfa9c9-51a1-475b-b821-c9d144ab8d2a","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:40:02.964Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"79bfa9c9-51a1-475b-b821-c9d144ab8d2a"}
{"level":"info","ts":"2026-11-03T15:40:02.964Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"79bfa9c9-51a1-475b-b821-c9d144ab8d2a","RayCluster":"ttl-waiting-kjf64","State":""}
{"level":"info","ts":"2026-11-03T15:40:02.964Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"79bfa9c9-51a1-475b-b821-c9d144ab8d2a","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:39:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:39:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:40:05.965Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"477ba059-0386-428f-8153-df95f6f66bdd","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-11-03T15:40:05.965Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"477ba059-0386-428f-8153-df95f6f66bdd","name":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:40:05.965Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"477ba059-0386-428f-8153-df95f6f66bdd","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:40:05.965Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"477ba059-0386-428f-8153-df95f6f66bdd"}
{"level":"info","ts":"2026-11-03T15:40:05.965Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"477ba059-0386-428f-8153-df95f6f66bdd","RayCluster":"ttl-waiting-kjf64","State":""}
{"level":"info","ts":"2026-11-03T15:40:05.965Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"477ba059-0386-428f-8153-df95f6f66bdd","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:39:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:39:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:40:08.966Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"9779e22d-c797-4a7b-9efb-22dd3b3027fe","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-11-03T15:40:08.966Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"9779e22d-c797-4a7b-9efb-22dd3b3027fe","name":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:40:08.966Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"9779e22d-c797-4a7b-9efb-22dd3b3027fe","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:40:08.966Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"9779e22d-c797-4a7b-9efb-22dd3b3027fe"}
{"level":"info","ts":"2026-11-03T15:40:08.966Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"9779e22d-c797-4a7b-9efb-22dd3b3027fe","RayCluster":"ttl-waiting-kjf64","State":""}
{"level":"info","ts":"2026-11-03T15:40:08.966Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"9779e22d-c797-4a7b-9efb-22dd3b3027fe","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:39:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:39:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:40:11.967Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"a11b8aea-25be-4015-a74f-f0a6040ccb8b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-11-03T15:40:11.967Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"a11b8aea-25be-4015-a74f-f0a6040ccb8b","name":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:40:11.967Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"a11b8aea-25be-4015-a74f-f0a6040ccb8b","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:40:11.967Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"a11b8aea-25be-4015-a74f-f0a6040ccb8b"}
{"level":"info","ts":"2026-11-03T15:40:11.967Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"a11b8aea-25be-4015-a74f-f0a6040ccb8b","RayCluster":"ttl-waiting-kjf64","State":""}
{"level":"info","ts":"2026-11-03T15:40:11.967Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"a11b8aea-25be-4015-a74f-f0a6040ccb8b","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:39:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:39:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:40:13.150Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"95b23c53-ca18-4c2c-ab41-d4990748f141"}
{"level":"info","ts":"2026-11-03T15:40:13.150Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"95b23c53-ca18-4c2c-ab41-d4990748f141"}
{"level":"info","ts":"2026-11-03T15:40:13.150Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"95b23c53-ca18-4c2c-ab41-d4990748f141","1 head service found":"ttl-waiting-kjf64-head-svc"}
{"level":"info","ts":"2026-11-03T15:40:13.150Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"95b23c53-ca18-4c2c-ab41-d4990748f141","Found 1 head Pod":"ttl-waiting-kjf64-head-dvbng","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:40:13.150Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"95b23c53-ca18-4c2c-ab41-d4990748f141","head Pod":"ttl-waiting-kjf64-head-dvbng","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ttl-waiting-kjf64-head-dvbng. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:40:13.150Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"95b23c53-ca18-4c2c-ab41-d4990748f141","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:40:13.150Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"95b23c53-ca18-4c2c-ab41-d4990748f141","worker Pod":"ttl-waiting-kjf64-small-group-worker-blcp8","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ttl-waiting-kjf64-small-group-worker-blcp8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:40:13.150Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"95b23c53-ca18-4c2c-ab41-d4990748f141","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:40:13.150Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"95b23c53-ca18-4c2c-ab41-d4990748f141","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:40:13.150Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"95b23c53-ca18-4c2c-ab41-d4990748f141","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:40:13.150Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"95b23c53-ca18-4c2c-ab41-d4990748f141","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:40:13.150Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"95b23c53-ca18-4c2c-ab41-d4990748f141","podName":"ttl-waiting-kjf64-head-dvbng","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:40:13.150Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"95b23c53-ca18-4c2c-ab41-d4990748f141","podName":"ttl-waiting-kjf64-head-dvbng","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:40:13.150Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"95b23c53-ca18-4c2c-ab41-d4990748f141","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:40:13.150Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"95b23c53-ca18-4c2c-ab41-d4990748f141","seconds":300}
{"level":"info","ts":"2026-11-03T15:40:14.153Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"50d770cc-f0e9-457c-9dfb-99c5bf8a41d9"}
{"level":"info","ts":"2026-11-03T15:40:14.153Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"50d770cc-f0e9-457c-9dfb-99c5bf8a41d9"}
{"level":"info","ts":"2026-11-03T15:40:14.153Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"50d770cc-f0e9-457c-9dfb-99c5bf8a41d9","1 head service found":"ttl-waiting-kjf64-head-svc"}
{"level":"info","ts":"2026-11-03T15:40:14.154Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"50d770cc-f0e9-457c-9dfb-99c5bf8a41d9","Found 1 head Pod":"ttl-waiting-kjf64-head-dvbng","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:40:14.154Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"50d770cc-f0e9-457c-9dfb-99c5bf8a41d9","head Pod":"ttl-waiting-kjf64-head-dvbng","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ttl-waiting-kjf64-head-dvbng. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:40:14.154Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"50d770cc-f0e9-457c-9dfb-99c5bf8a41d9","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:40:14.154Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"50d770cc-f0e9-457c-9dfb-99c5bf8a41d9","worker Pod":"ttl-waiting-kjf64-small-group-worker-blcp8","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ttl-waiting-kjf64-small-group-worker-blcp8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:40:14.154Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"50d770cc-f0e9-457c-9dfb-99c5bf8a41d9","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:40:14.154Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"50d770cc-f0e9-457c-9dfb-99c5bf8a41d9","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:40:14.154Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"50d770cc-f0e9-457c-9dfb-99c5bf8a41d9","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:40:14.154Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"50d770cc-f0e9-457c-9dfb-99c5bf8a41d9","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:40:14.154Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"50d770cc-f0e9-457c-9dfb-99c5bf8a41d9","podName":"ttl-waiting-kjf64-head-dvbng","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:39:01Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:40:14.154Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"50d770cc-f0e9-457c-9dfb-99c5bf8a41d9","podName":"ttl-waiting-kjf64-head-dvbng","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:39:01Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:40:14.154Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"50d770cc-f0e9-457c-9dfb-99c5bf8a41d9","name":"ttl-waiting-kjf64","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:39:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:40:14Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:40:14.215Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"7bc653c3-b955-4c10-a26b-d9c9334a8be9","cluster":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:40:14.215Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"688f708a-f6e6-4a34-a25e-5de7522fd62d","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-11-03T15:40:14.215Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"688f708a-f6e6-4a34-a25e-5de7522fd62d","name":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:40:14.215Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"688f708a-f6e6-4a34-a25e-5de7522fd62d","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:40:14.215Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"688f708a-f6e6-4a34-a25e-5de7522fd62d"}
{"level":"info","ts":"2026-11-03T15:40:14.215Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"688f708a-f6e6-4a34-a25e-5de7522fd62d","RayCluster":"ttl-waiting-kjf64","State":""}
{"level":"info","ts":"2026-11-03T15:40:14.215Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"688f708a-f6e6-4a34-a25e-5de7522fd62d","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:39:02Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:40:14Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:40:14.215Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"688f708a-f6e6-4a34-a25e-5de7522fd62d","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:40:14.276Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"daa748a7-b9c0-473c-a264-e9f081f0c462","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-11-03T15:40:14.276Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"daa748a7-b9c0-473c-a264-e9f081f0c462","name":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:40:14.277Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"daa748a7-b9c0-473c-a264-e9f081f0c462","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:40:14.277Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"daa748a7-b9c0-473c-a264-e9f081f0c462"}
{"level":"info","ts":"2026-11-03T15:40:14.277Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"daa748a7-b9c0-473c-a264-e9f081f0c462","RayCluster":"ttl-waiting-kjf64","State":""}
{"level":"info","ts":"2026-11-03T15:40:14.277Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"daa748a7-b9c0-473c-a264-e9f081f0c462","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:40:14Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:40:14Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:40:16.215Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"9b662464-2c66-4c04-b282-5a9022a29afa"}
{"level":"info","ts":"2026-11-03T15:40:16.215Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"9b662464-2c66-4c04-b282-5a9022a29afa"}
{"level":"info","ts":"2026-11-03T15:40:16.215Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"9b662464-2c66-4c04-b282-5a9022a29afa","1 head service found":"ttl-waiting-kjf64-head-svc"}
{"level":"info","ts":"2026-11-03T15:40:16.215Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"9b662464-2c66-4c04-b282-5a9022a29afa","Found 1 head Pod":"ttl-waiting-kjf64-head-dvbng","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:40:16.215Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"9b662464-2c66-4c04-b282-5a9022a29afa","head Pod":"ttl-waiting-kjf64-head-dvbng","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ttl-waiting-kjf64-head-dvbng. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:40:16.215Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"9b662464-2c66-4c04-b282-5a9022a29afa","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:40:16.215Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"9b662464-2c66-4c04-b282-5a9022a29afa","worker Pod":"ttl-waiting-kjf64-small-group-worker-blcp8","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ttl-waiting-kjf64-small-group-worker-blcp8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:40:16.216Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"9b662464-2c66-4c04-b282-5a9022a29afa","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:40:16.216Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"9b662464-2c66-4c04-b282-5a9022a29afa","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:40:16.216Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"9b662464-2c66-4c04-b282-5a9022a29afa","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:40:16.216Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"9b662464-2c66-4c04-b282-5a9022a29afa","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:40:16.216Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"9b662464-2c66-4c04-b282-5a9022a29afa","podName":"ttl-waiting-kjf64-small-group-worker-blcp8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:40:16.216Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"9b662464-2c66-4c04-b282-5a9022a29afa","podName":"ttl-waiting-kjf64-small-group-worker-blcp8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:40:16.216Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"9b662464-2c66-4c04-b282-5a9022a29afa","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:40:16.216Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"9b662464-2c66-4c04-b282-5a9022a29afa","seconds":300}
{"level":"info","ts":"2026-11-03T15:40:17.278Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"de9371f1-3f87-4d88-b7d0-3ea0e9e8f6f9","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-11-03T15:40:17.278Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"de9371f1-3f87-4d88-b7d0-3ea0e9e8f6f9","name":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:40:17.278Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"de9371f1-3f87-4d88-b7d0-3ea0e9e8f6f9","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:40:17.278Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"de9371f1-3f87-4d88-b7d0-3ea0e9e8f6f9"}
{"level":"info","ts":"2026-11-03T15:40:17.278Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"de9371f1-3f87-4d88-b7d0-3ea0e9e8f6f9","RayCluster":"ttl-waiting-kjf64","State":""}
{"level":"info","ts":"2026-11-03T15:40:17.278Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"de9371f1-3f87-4d88-b7d0-3ea0e9e8f6f9","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:40:14Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:40:14Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:40:20.278Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"27aab481-f68c-48b5-a3ff-e45cd72f1db5","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-11-03T15:40:20.278Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"27aab481-f68c-48b5-a3ff-e45cd72f1db5","name":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:40:20.279Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"27aab481-f68c-48b5-a3ff-e45cd72f1db5","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:40:20.279Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"27aab481-f68c-48b5-a3ff-e45cd72f1db5"}
{"level":"info","ts":"2026-11-03T15:40:20.279Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"27aab481-f68c-48b5-a3ff-e45cd72f1db5","RayCluster":"ttl-waiting-kjf64","State":""}
{"level":"info","ts":"2026-11-03T15:40:20.279Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"27aab481-f68c-48b5-a3ff-e45cd72f1db5","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:40:14Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:40:14Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:40:23.279Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"09d9f574-91a9-4ef3-8772-6fd4d2a88970","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-11-03T15:40:23.279Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"09d9f574-91a9-4ef3-8772-6fd4d2a88970","name":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:40:23.279Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"09d9f574-91a9-4ef3-8772-6fd4d2a88970","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:40:23.280Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"09d9f574-91a9-4ef3-8772-6fd4d2a88970"}
{"level":"info","ts":"2026-11-03T15:40:23.280Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"09d9f574-91a9-4ef3-8772-6fd4d2a88970","RayCluster":"ttl-waiting-kjf64","State":""}
{"level":"info","ts":"2026-11-03T15:40:23.280Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"09d9f574-91a9-4ef3-8772-6fd4d2a88970","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:40:14Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:40:14Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:40:25.213Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"a115ab0e-1a53-4b5d-8d6b-00d7e630d18d"}
{"level":"info","ts":"2026-11-03T15:40:25.213Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"a115ab0e-1a53-4b5d-8d6b-00d7e630d18d"}
{"level":"info","ts":"2026-11-03T15:40:25.213Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"a115ab0e-1a53-4b5d-8d6b-00d7e630d18d","1 head service found":"ttl-waiting-kjf64-head-svc"}
{"level":"info","ts":"2026-11-03T15:40:25.213Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"a115ab0e-1a53-4b5d-8d6b-00d7e630d18d","Found 1 head Pod":"ttl-waiting-kjf64-head-dvbng","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:40:25.213Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"a115ab0e-1a53-4b5d-8d6b-00d7e630d18d","head Pod":"ttl-waiting-kjf64-head-dvbng","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ttl-waiting-kjf64-head-dvbng. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:40:25.213Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"a115ab0e-1a53-4b5d-8d6b-00d7e630d18d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:40:25.213Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"a115ab0e-1a53-4b5d-8d6b-00d7e630d18d","worker Pod":"ttl-waiting-kjf64-small-group-worker-blcp8","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ttl-waiting-kjf64-small-group-worker-blcp8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:40:25.213Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"a115ab0e-1a53-4b5d-8d6b-00d7e630d18d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:40:25.213Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"a115ab0e-1a53-4b5d-8d6b-00d7e630d18d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:40:25.213Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"a115ab0e-1a53-4b5d-8d6b-00d7e630d18d","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:40:25.213Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"a115ab0e-1a53-4b5d-8d6b-00d7e630d18d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:40:25.213Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"a115ab0e-1a53-4b5d-8d6b-00d7e630d18d","podName":"ttl-waiting-kjf64-small-group-worker-blcp8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:40:25.213Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"a115ab0e-1a53-4b5d-8d6b-00d7e630d18d","podName":"ttl-waiting-kjf64-small-group-worker-blcp8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:40:25.213Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"a115ab0e-1a53-4b5d-8d6b-00d7e630d18d","name":"ttl-waiting-kjf64","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:40:14Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:40:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:40:25Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:40:25.274Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"4b88f741-a50a-4984-b94d-e08a37c4b9fe","cluster":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:40:25.274Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"6e9c31da-8d7d-4c98-9125-90e2c8aea6b6","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-11-03T15:40:25.274Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"6e9c31da-8d7d-4c98-9125-90e2c8aea6b6","name":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:40:25.274Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"6e9c31da-8d7d-4c98-9125-90e2c8aea6b6","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:40:25.274Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"6e9c31da-8d7d-4c98-9125-90e2c8aea6b6"}
{"level":"info","ts":"2026-11-03T15:40:25.274Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"6e9c31da-8d7d-4c98-9125-90e2c8aea6b6","RayCluster":"ttl-waiting-kjf64","State":""}
{"level":"info","ts":"2026-11-03T15:40:25.274Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"6e9c31da-8d7d-4c98-9125-90e2c8aea6b6","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:40:14Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:40:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:40:25Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:40:25.274Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"6e9c31da-8d7d-4c98-9125-90e2c8aea6b6","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:40:25.334Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"2ebcb140-4a27-499e-bf8a-bdc9efd72809","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-11-03T15:40:25.334Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"2ebcb140-4a27-499e-bf8a-bdc9efd72809","name":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:40:25.334Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"2ebcb140-4a27-499e-bf8a-bdc9efd72809","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:40:25.335Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"2ebcb140-4a27-499e-bf8a-bdc9efd72809"}
{"level":"info","ts":"2026-11-03T15:40:25.335Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"2ebcb140-4a27-499e-bf8a-bdc9efd72809","RayCluster":"ttl-waiting-kjf64","State":""}
{"level":"info","ts":"2026-11-03T15:40:25.335Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"2ebcb140-4a27-499e-bf8a-bdc9efd72809","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:40:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:40:25Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:40:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:40:25Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:40:27.274Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"ec30175b-4047-47fc-a74e-dd0b95036ae3"}
{"level":"info","ts":"2026-11-03T15:40:27.274Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"ec30175b-4047-47fc-a74e-dd0b95036ae3"}
{"level":"info","ts":"2026-11-03T15:40:27.274Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"ec30175b-4047-47fc-a74e-dd0b95036ae3","1 head service found":"ttl-waiting-kjf64-head-svc"}
{"level":"info","ts":"2026-11-03T15:40:27.274Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"ec30175b-4047-47fc-a74e-dd0b95036ae3","Found 1 head Pod":"ttl-waiting-kjf64-head-dvbng","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:40:27.274Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"ec30175b-4047-47fc-a74e-dd0b95036ae3","head Pod":"ttl-waiting-kjf64-head-dvbng","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ttl-waiting-kjf64-head-dvbng. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:40:27.274Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"ec30175b-4047-47fc-a74e-dd0b95036ae3","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:40:27.274Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"ec30175b-4047-47fc-a74e-dd0b95036ae3","worker Pod":"ttl-waiting-kjf64-small-group-worker-blcp8","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ttl-waiting-kjf64-small-group-worker-blcp8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:40:27.274Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"ec30175b-4047-47fc-a74e-dd0b95036ae3","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:40:27.274Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"ec30175b-4047-47fc-a74e-dd0b95036ae3","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:40:27.274Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"ec30175b-4047-47fc-a74e-dd0b95036ae3","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:40:27.274Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"ec30175b-4047-47fc-a74e-dd0b95036ae3","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:40:27.274Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"ec30175b-4047-47fc-a74e-dd0b95036ae3","podName":"ttl-waiting-kjf64-small-group-worker-blcp8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:40:27.274Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"ec30175b-4047-47fc-a74e-dd0b95036ae3","podName":"ttl-waiting-kjf64-small-group-worker-blcp8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:40:27.274Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"ec30175b-4047-47fc-a74e-dd0b95036ae3","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:40:27.274Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"ec30175b-4047-47fc-a74e-dd0b95036ae3","seconds":300}
{"level":"info","ts":"2026-11-03T15:40:28.335Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"66ca12e3-b9a9-48a1-b650-290146c14f25","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-11-03T15:40:28.335Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"66ca12e3-b9a9-48a1-b650-290146c14f25","name":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:40:28.335Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"66ca12e3-b9a9-48a1-b650-290146c14f25","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:40:28.335Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"66ca12e3-b9a9-48a1-b650-290146c14f25"}
{"level":"info","ts":"2026-11-03T15:40:28.335Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"66ca12e3-b9a9-48a1-b650-290146c14f25","RayCluster":"ttl-waiting-kjf64","State":""}
{"level":"info","ts":"2026-11-03T15:40:28.335Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"66ca12e3-b9a9-48a1-b650-290146c14f25","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:40:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:40:25Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:40:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:40:25Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:40:29.494Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"47a8e1be-caa4-4d9c-a96d-1a859fc2077d"}
{"level":"info","ts":"2026-11-03T15:40:29.494Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"47a8e1be-caa4-4d9c-a96d-1a859fc2077d"}
{"level":"info","ts":"2026-11-03T15:40:29.494Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"47a8e1be-caa4-4d9c-a96d-1a859fc2077d","1 head service found":"ttl-waiting-kjf64-head-svc"}
{"level":"info","ts":"2026-11-03T15:40:29.494Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"47a8e1be-caa4-4d9c-a96d-1a859fc2077d","Found 1 head Pod":"ttl-waiting-kjf64-head-dvbng","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:40:29.494Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"47a8e1be-caa4-4d9c-a96d-1a859fc2077d","head Pod":"ttl-waiting-kjf64-head-dvbng","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ttl-waiting-kjf64-head-dvbng. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:40:29.494Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"47a8e1be-caa4-4d9c-a96d-1a859fc2077d","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:40:29.494Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"47a8e1be-caa4-4d9c-a96d-1a859fc2077d","worker Pod":"ttl-waiting-kjf64-small-group-worker-blcp8","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ttl-waiting-kjf64-small-group-worker-blcp8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:40:29.494Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"47a8e1be-caa4-4d9c-a96d-1a859fc2077d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:40:29.494Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"47a8e1be-caa4-4d9c-a96d-1a859fc2077d","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:40:29.494Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"47a8e1be-caa4-4d9c-a96d-1a859fc2077d","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:40:29.494Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"47a8e1be-caa4-4d9c-a96d-1a859fc2077d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:40:29.494Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"47a8e1be-caa4-4d9c-a96d-1a859fc2077d","podName":"ttl-waiting-kjf64-small-group-worker-blcp8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:40:29.494Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"47a8e1be-caa4-4d9c-a96d-1a859fc2077d","podName":"ttl-waiting-kjf64-small-group-worker-blcp8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:40:29.494Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"47a8e1be-caa4-4d9c-a96d-1a859fc2077d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:40:29.494Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"47a8e1be-caa4-4d9c-a96d-1a859fc2077d","seconds":300}
{"level":"info","ts":"2026-11-03T15:40:30.547Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"b108634d-cc05-4922-92d9-fe7aafe3717a"}
{"level":"info","ts":"2026-11-03T15:40:30.547Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"b108634d-cc05-4922-92d9-fe7aafe3717a"}
{"level":"info","ts":"2026-11-03T15:40:30.547Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"b108634d-cc05-4922-92d9-fe7aafe3717a","1 head service found":"ttl-waiting-kjf64-head-svc"}
{"level":"info","ts":"2026-11-03T15:40:30.547Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"b108634d-cc05-4922-92d9-fe7aafe3717a","Found 1 head Pod":"ttl-waiting-kjf64-head-dvbng","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:40:30.547Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"b108634d-cc05-4922-92d9-fe7aafe3717a","head Pod":"ttl-waiting-kjf64-head-dvbng","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ttl-waiting-kjf64-head-dvbng. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:40:30.547Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"b108634d-cc05-4922-92d9-fe7aafe3717a","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:40:30.547Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"b108634d-cc05-4922-92d9-fe7aafe3717a","worker Pod":"ttl-waiting-kjf64-small-group-worker-blcp8","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ttl-waiting-kjf64-small-group-worker-blcp8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:40:30.547Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"b108634d-cc05-4922-92d9-fe7aafe3717a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:40:30.547Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"b108634d-cc05-4922-92d9-fe7aafe3717a","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:40:30.547Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"b108634d-cc05-4922-92d9-fe7aafe3717a","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:40:30.547Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"b108634d-cc05-4922-92d9-fe7aafe3717a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:40:30.547Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"b108634d-cc05-4922-92d9-fe7aafe3717a","podName":"ttl-waiting-kjf64-small-group-worker-blcp8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:40:30.547Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"b108634d-cc05-4922-92d9-fe7aafe3717a","podName":"ttl-waiting-kjf64-small-group-worker-blcp8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:40:30.547Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"b108634d-cc05-4922-92d9-fe7aafe3717a","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:40:30.547Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"b108634d-cc05-4922-92d9-fe7aafe3717a","seconds":300}
{"level":"info","ts":"2026-11-03T15:40:31.336Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"d33f2821-cfd1-4c39-b8b6-54138e165494","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-11-03T15:40:31.336Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"d33f2821-cfd1-4c39-b8b6-54138e165494","name":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:40:31.336Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"d33f2821-cfd1-4c39-b8b6-54138e165494","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:40:31.336Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"d33f2821-cfd1-4c39-b8b6-54138e165494"}
{"level":"info","ts":"2026-11-03T15:40:31.336Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"d33f2821-cfd1-4c39-b8b6-54138e165494","RayCluster":"ttl-waiting-kjf64","State":""}
{"level":"info","ts":"2026-11-03T15:40:31.336Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"d33f2821-cfd1-4c39-b8b6-54138e165494","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:40:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:40:25Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:40:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:40:25Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:40:34.337Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"2c1715f9-bcec-4f08-bd3d-c17ffdc00304","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-11-03T15:40:34.337Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"2c1715f9-bcec-4f08-bd3d-c17ffdc00304","name":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:40:34.337Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"2c1715f9-bcec-4f08-bd3d-c17ffdc00304","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:40:34.337Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"2c1715f9-bcec-4f08-bd3d-c17ffdc00304"}
{"level":"info","ts":"2026-11-03T15:40:34.337Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"2c1715f9-bcec-4f08-bd3d-c17ffdc00304","RayCluster":"ttl-waiting-kjf64","State":""}
{"level":"info","ts":"2026-11-03T15:40:34.337Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"2c1715f9-bcec-4f08-bd3d-c17ffdc00304","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:40:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:40:25Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:40:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:40:25Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:40:37.337Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"ac4a2e3f-244c-4968-bcaa-af920cea7aec","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-11-03T15:40:37.337Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"ac4a2e3f-244c-4968-bcaa-af920cea7aec","name":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:40:37.338Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"ac4a2e3f-244c-4968-bcaa-af920cea7aec","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:40:37.338Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"ac4a2e3f-244c-4968-bcaa-af920cea7aec"}
{"level":"info","ts":"2026-11-03T15:40:37.338Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"ac4a2e3f-244c-4968-bcaa-af920cea7aec","RayCluster":"ttl-waiting-kjf64","State":""}
{"level":"info","ts":"2026-11-03T15:40:37.338Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"ac4a2e3f-244c-4968-bcaa-af920cea7aec","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:40:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:40:25Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:40:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:40:25Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:40:39.540Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"da94ca81-bd91-436b-a39a-b305975faded"}
{"level":"info","ts":"2026-11-03T15:40:39.540Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"da94ca81-bd91-436b-a39a-b305975faded"}
{"level":"info","ts":"2026-11-03T15:40:39.540Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"da94ca81-bd91-436b-a39a-b305975faded","1 head service found":"ttl-waiting-kjf64-head-svc"}
{"level":"info","ts":"2026-11-03T15:40:39.541Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"da94ca81-bd91-436b-a39a-b305975faded","Found 1 head Pod":"ttl-waiting-kjf64-head-dvbng","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:40:39.541Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"da94ca81-bd91-436b-a39a-b305975faded","head Pod":"ttl-waiting-kjf64-head-dvbng","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ttl-waiting-kjf64-head-dvbng. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:40:39.541Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"da94ca81-bd91-436b-a39a-b305975faded","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:40:39.541Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"da94ca81-bd91-436b-a39a-b305975faded","worker Pod":"ttl-waiting-kjf64-small-group-worker-blcp8","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ttl-waiting-kjf64-small-group-worker-blcp8. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:40:39.541Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"da94ca81-bd91-436b-a39a-b305975faded","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:40:39.541Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"da94ca81-bd91-436b-a39a-b305975faded","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:40:39.541Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"da94ca81-bd91-436b-a39a-b305975faded","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:40:39.541Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"da94ca81-bd91-436b-a39a-b305975faded","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:40:39.541Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"da94ca81-bd91-436b-a39a-b305975faded","podName":"ttl-waiting-kjf64-small-group-worker-blcp8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:40:39.541Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"da94ca81-bd91-436b-a39a-b305975faded","podName":"ttl-waiting-kjf64-small-group-worker-blcp8","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:40:39.541Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"da94ca81-bd91-436b-a39a-b305975faded","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:40:39.541Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"da94ca81-bd91-436b-a39a-b305975faded","seconds":300}
{"level":"info","ts":"2026-11-03T15:40:40.338Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"03d54517-7bb1-437b-9d61-ed0e14af3564","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-11-03T15:40:40.338Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"03d54517-7bb1-437b-9d61-ed0e14af3564","name":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:40:40.338Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"03d54517-7bb1-437b-9d61-ed0e14af3564","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:40:40.338Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"03d54517-7bb1-437b-9d61-ed0e14af3564"}
{"level":"info","ts":"2026-11-03T15:40:40.338Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"03d54517-7bb1-437b-9d61-ed0e14af3564","RayCluster":"ttl-waiting-kjf64","State":""}
{"level":"info","ts":"2026-11-03T15:40:40.338Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"03d54517-7bb1-437b-9d61-ed0e14af3564","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:40:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:40:25Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:40:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:40:25Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:40:40.594Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"b35b9688-ed18-4972-8c01-adf0c009bec6"}
{"level":"info","ts":"2026-11-03T15:40:40.594Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"b35b9688-ed18-4972-8c01-adf0c009bec6"}
{"level":"info","ts":"2026-11-03T15:40:40.595Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"b35b9688-ed18-4972-8c01-adf0c009bec6","1 head service found":"ttl-waiting-kjf64-head-svc"}
{"level":"info","ts":"2026-11-03T15:40:40.595Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"b35b9688-ed18-4972-8c01-adf0c009bec6","Found 1 head Pod":"ttl-waiting-kjf64-head-dvbng","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:40:40.595Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"b35b9688-ed18-4972-8c01-adf0c009bec6","head Pod":"ttl-waiting-kjf64-head-dvbng","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ttl-waiting-kjf64-head-dvbng. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:40:40.595Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"b35b9688-ed18-4972-8c01-adf0c009bec6","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:40:40.595Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"b35b9688-ed18-4972-8c01-adf0c009bec6","worker Pod":"ttl-waiting-kjf64-small-group-worker-blcp8","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ttl-waiting-kjf64-small-group-worker-blcp8. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:40:40.595Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"b35b9688-ed18-4972-8c01-adf0c009bec6","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:40:40.595Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"b35b9688-ed18-4972-8c01-adf0c009bec6","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:40:40.595Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"b35b9688-ed18-4972-8c01-adf0c009bec6","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:40:40.595Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"b35b9688-ed18-4972-8c01-adf0c009bec6","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:40:40.595Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"b35b9688-ed18-4972-8c01-adf0c009bec6","podName":"ttl-waiting-kjf64-small-group-worker-blcp8","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:39:01Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:40:40.595Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"b35b9688-ed18-4972-8c01-adf0c009bec6","podName":"ttl-waiting-kjf64-small-group-worker-blcp8","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:39:01Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:40:40.595Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"b35b9688-ed18-4972-8c01-adf0c009bec6","name":"ttl-waiting-kjf64","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:40:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:40:25Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:40:40Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:40:25Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:40:40.656Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"1e902166-0617-4a44-868f-b80f2484d1c5","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-11-03T15:40:40.656Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"f5fc02e9-aeb7-401f-abfa-a5cc23481f9f","cluster":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:40:40.656Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"1e902166-0617-4a44-868f-b80f2484d1c5","name":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:40:40.656Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"1e902166-0617-4a44-868f-b80f2484d1c5","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:40:40.656Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"1e902166-0617-4a44-868f-b80f2484d1c5"}
{"level":"info","ts":"2026-11-03T15:40:40.656Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"1e902166-0617-4a44-868f-b80f2484d1c5","RayCluster":"ttl-waiting-kjf64","State":""}
{"level":"info","ts":"2026-11-03T15:40:40.656Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"1e902166-0617-4a44-868f-b80f2484d1c5","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:40:25Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:40:25Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:40:40Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:40:25Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:40:40.656Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"1e902166-0617-4a44-868f-b80f2484d1c5","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Initializing"}
{"level":"info","ts":"2026-11-03T15:40:40.717Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"8a09e511-9e51-4d9c-aafd-1f1567cd53ac","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-11-03T15:40:40.717Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"8a09e511-9e51-4d9c-aafd-1f1567cd53ac","name":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:40:40.717Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"8a09e511-9e51-4d9c-aafd-1f1567cd53ac","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:40:40.717Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"8a09e511-9e51-4d9c-aafd-1f1567cd53ac"}
{"level":"info","ts":"2026-11-03T15:40:40.717Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"8a09e511-9e51-4d9c-aafd-1f1567cd53ac","RayCluster":"ttl-waiting-kjf64","State":""}
{"level":"info","ts":"2026-11-03T15:40:40.717Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"8a09e511-9e51-4d9c-aafd-1f1567cd53ac","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:40:40Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:40:25Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:40:40Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:40:25Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:40:42.660Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"21f4bf11-cede-4ca3-849e-6f737c9cbdbb"}
{"level":"info","ts":"2026-11-03T15:40:42.660Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"21f4bf11-cede-4ca3-849e-6f737c9cbdbb"}
{"level":"info","ts":"2026-11-03T15:40:42.660Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"21f4bf11-cede-4ca3-849e-6f737c9cbdbb","1 head service found":"ttl-waiting-kjf64-head-svc"}
{"level":"info","ts":"2026-11-03T15:40:42.660Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"21f4bf11-cede-4ca3-849e-6f737c9cbdbb","Found 1 head Pod":"ttl-waiting-kjf64-head-dvbng","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:40:42.660Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"21f4bf11-cede-4ca3-849e-6f737c9cbdbb","head Pod":"ttl-waiting-kjf64-head-dvbng","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ttl-waiting-kjf64-head-dvbng. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:40:42.660Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"21f4bf11-cede-4ca3-849e-6f737c9cbdbb","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:40:42.660Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"21f4bf11-cede-4ca3-849e-6f737c9cbdbb","worker Pod":"ttl-waiting-kjf64-small-group-worker-blcp8","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ttl-waiting-kjf64-small-group-worker-blcp8. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:40:42.660Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"21f4bf11-cede-4ca3-849e-6f737c9cbdbb","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:40:42.660Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"21f4bf11-cede-4ca3-849e-6f737c9cbdbb","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:40:42.660Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"21f4bf11-cede-4ca3-849e-6f737c9cbdbb","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:40:42.660Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"21f4bf11-cede-4ca3-849e-6f737c9cbdbb","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:40:42.660Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"21f4bf11-cede-4ca3-849e-6f737c9cbdbb","podName":"ttl-waiting-kjf64-small-group-worker-blcp8","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:39:01Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:40:42.660Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"21f4bf11-cede-4ca3-849e-6f737c9cbdbb","podName":"ttl-waiting-kjf64-small-group-worker-blcp8","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:39:01Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:40:42.660Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"21f4bf11-cede-4ca3-849e-6f737c9cbdbb","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:40:42.660Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"21f4bf11-cede-4ca3-849e-6f737c9cbdbb","seconds":300}
{"level":"info","ts":"2026-11-03T15:40:43.719Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"2dd10fb3-a03e-4f82-a779-73c8d2b362fd","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-11-03T15:40:43.719Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"2dd10fb3-a03e-4f82-a779-73c8d2b362fd","name":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:40:43.719Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"2dd10fb3-a03e-4f82-a779-73c8d2b362fd","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:40:43.719Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"2dd10fb3-a03e-4f82-a779-73c8d2b362fd"}
{"level":"info","ts":"2026-11-03T15:40:43.719Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"2dd10fb3-a03e-4f82-a779-73c8d2b362fd","RayCluster":"ttl-waiting-kjf64","State":""}
{"level":"info","ts":"2026-11-03T15:40:43.719Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"2dd10fb3-a03e-4f82-a779-73c8d2b362fd","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:40:40Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:40:25Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:40:40Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:40:25Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:40:46.720Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"317158a4-d90c-4d92-a457-3ca31b293d4b","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-11-03T15:40:46.720Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"317158a4-d90c-4d92-a457-3ca31b293d4b","name":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:40:46.720Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"317158a4-d90c-4d92-a457-3ca31b293d4b","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:40:46.720Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"317158a4-d90c-4d92-a457-3ca31b293d4b"}
{"level":"info","ts":"2026-11-03T15:40:46.720Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"317158a4-d90c-4d92-a457-3ca31b293d4b","RayCluster":"ttl-waiting-kjf64","State":""}
{"level":"info","ts":"2026-11-03T15:40:46.720Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"317158a4-d90c-4d92-a457-3ca31b293d4b","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:40:40Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:40:25Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:40:40Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:40:25Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:40:49.721Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"a4fea379-9a7d-46a2-bf70-f0723a9f64d7","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-11-03T15:40:49.721Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"a4fea379-9a7d-46a2-bf70-f0723a9f64d7","name":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:40:49.721Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"a4fea379-9a7d-46a2-bf70-f0723a9f64d7","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:40:49.721Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"a4fea379-9a7d-46a2-bf70-f0723a9f64d7"}
{"level":"info","ts":"2026-11-03T15:40:49.721Z","logger":"controllers.RayJob","msg":"Wait for the RayCluster.Status.State to be ready before submitting the job.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"a4fea379-9a7d-46a2-bf70-f0723a9f64d7","RayCluster":"ttl-waiting-kjf64","State":""}
{"level":"info","ts":"2026-11-03T15:40:49.721Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"a4fea379-9a7d-46a2-bf70-f0723a9f64d7","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:40:40Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:40:25Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:40:40Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:40:25Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:40:51.556Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"5aea5f26-2d16-4045-83f0-bfcf8d03a782"}
{"level":"info","ts":"2026-11-03T15:40:51.556Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"5aea5f26-2d16-4045-83f0-bfcf8d03a782"}
{"level":"info","ts":"2026-11-03T15:40:51.556Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"5aea5f26-2d16-4045-83f0-bfcf8d03a782","1 head service found":"ttl-waiting-kjf64-head-svc"}
{"level":"info","ts":"2026-11-03T15:40:51.556Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"5aea5f26-2d16-4045-83f0-bfcf8d03a782","Found 1 head Pod":"ttl-waiting-kjf64-head-dvbng","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:40:51.556Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"5aea5f26-2d16-4045-83f0-bfcf8d03a782","head Pod":"ttl-waiting-kjf64-head-dvbng","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ttl-waiting-kjf64-head-dvbng. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:40:51.556Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"5aea5f26-2d16-4045-83f0-bfcf8d03a782","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:40:51.556Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"5aea5f26-2d16-4045-83f0-bfcf8d03a782","worker Pod":"ttl-waiting-kjf64-small-group-worker-blcp8","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ttl-waiting-kjf64-small-group-worker-blcp8. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:40:51.556Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"5aea5f26-2d16-4045-83f0-bfcf8d03a782","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:40:51.556Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"5aea5f26-2d16-4045-83f0-bfcf8d03a782","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:40:51.556Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"5aea5f26-2d16-4045-83f0-bfcf8d03a782","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:40:51.556Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"5aea5f26-2d16-4045-83f0-bfcf8d03a782","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:40:51.556Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"5aea5f26-2d16-4045-83f0-bfcf8d03a782","name":"ttl-waiting-kjf64","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:40:40Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:40:25Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:40:51Z","stateTransitionTimes":{"ready":"2026-11-03T15:40:51Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:40:25Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:40:51Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:40:51.618Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"78db8c9f-3ea9-4b22-a00a-937373141d9d","cluster":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:40:51.618Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"766b1f8f-fef1-4ea7-a995-16c60477f550","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-11-03T15:40:51.618Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"766b1f8f-fef1-4ea7-a995-16c60477f550","name":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:40:51.618Z","logger":"controllers.RayJob","msg":"Found the associated RayCluster for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"766b1f8f-fef1-4ea7-a995-16c60477f550","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:40:51.618Z","logger":"controllers.RayJob","msg":"Disregard changes in RayClusterSpec of RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"766b1f8f-fef1-4ea7-a995-16c60477f550"}
{"level":"info","ts":"2026-11-03T15:40:51.618Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"766b1f8f-fef1-4ea7-a995-16c60477f550","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","jobDeploymentStatus":"Initializing","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:40:40Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:40:25Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","dashboardURL":"ttl-waiting-kjf64-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobDeploymentStatus":"Waiting","startTime":"2026-11-03T15:38:58Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:40:40Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:40:25Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:40:51.618Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"766b1f8f-fef1-4ea7-a995-16c60477f550","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Initializing","new JobDeploymentStatus":"Waiting"}
{"level":"info","ts":"2026-11-03T15:40:51.678Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"990d61ee-2e6f-4fad-ba5f-5abcce81aa74","JobStatus":"","JobDeploymentStatus":"Waiting","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-11-03T15:40:52.194Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"1b475bec-0256-4c45-9c20-09b6b0967300","JobStatus":"","JobDeploymentStatus":"Waiting","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-11-03T15:40:52.194Z","logger":"controllers.RayJob","msg":"The RayJob has passed the preRunningDeadlineSeconds. Transition the status to `Failed`.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"1b475bec-0256-4c45-9c20-09b6b0967300","StartTime":"2026-11-03 15:35:47 +0000 UTC","PreRunningDeadlineSeconds":300}
{"level":"info","ts":"2026-11-03T15:40:52.194Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"1b475bec-0256-4c45-9c20-09b6b0967300","oldRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","dashboardURL":"ttl-waiting-kjf64-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobDeploymentStatus":"Waiting","startTime":"2026-11-03T15:35:47Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:40:40Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:40:25Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}},"newRayJobStatus":{"rayJobInfo":{},"rayClusterName":"ttl-waiting-kjf64","dashboardURL":"ttl-waiting-kjf64-head-svc.test-ns-kbx4w.svc.cluster.local:8265","jobDeploymentStatus":"Failed","reason":"PreRunningDeadlineExceeded","message":"The RayJob has passed the preRunningDeadlineSeconds. StartTime: 2026-11-03 15:35:47 +0000 UTC. PreRunningDeadlineSeconds: 300","startTime":"2026-11-03T15:35:47Z","succeeded":0,"failed":1,"rayClusterStatus":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:40:40Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.51","serviceIP":"10.132.0.51","podName":"ttl-waiting-kjf64-head-dvbng","serviceName":"ttl-waiting-kjf64-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:40:25Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:38:58Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:38:59Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}}
{"level":"info","ts":"2026-11-03T15:40:52.194Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"1b475bec-0256-4c45-9c20-09b6b0967300","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"Waiting","new JobDeploymentStatus":"Failed"}
{"level":"info","ts":"2026-11-03T15:40:52.255Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"b706009c-70ba-4c36-a14e-bc07812a6bd2","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-11-03T15:40:52.255Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"b706009c-70ba-4c36-a14e-bc07812a6bd2","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Failed","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-11-03 15:40:52 +0000 UTC","Now":"2026-11-03T15:40:52.255Z","ShutdownTime":"2026-11-03T15:40:52.000Z"}
{"level":"info","ts":"2026-11-03T15:40:52.312Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob is deleted","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"b706009c-70ba-4c36-a14e-bc07812a6bd2","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:40:52.312Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"b706009c-70ba-4c36-a14e-bc07812a6bd2","isClusterDeleted":false}
{"level":"info","ts":"2026-11-03T15:40:52.312Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"b706009c-70ba-4c36-a14e-bc07812a6bd2","deletionMechanism":"ShutdownAfterJobFinishes","RayCluster":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:40:52.313Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"reconcileID":"9ba7c6d7-cda6-4151-ae89-25ca10d1058b"}
{"level":"info","ts":"2026-11-03T15:40:52.313Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"0da038df-429a-4a51-807e-9c584d6afaf1","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-11-03T15:40:52.313Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-waiting-kjf64","reconcileID":"2924cb28-fc5a-44f0-beb9-0cf9105fab39","namespacedName":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:40:52.313Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"0da038df-429a-4a51-807e-9c584d6afaf1","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Failed","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-11-03 15:40:52 +0000 UTC","Now":"2026-11-03T15:40:52.313Z","ShutdownTime":"2026-11-03T15:40:52.000Z"}
{"level":"info","ts":"2026-11-03T15:40:52.313Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-waiting-kjf64","reconcileID":"2924cb28-fc5a-44f0-beb9-0cf9105fab39","cluster":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:40:52.313Z","logger":"controllers.RayJob","msg":"The deletion of the associated RayCluster for RayJob is ongoing.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"0da038df-429a-4a51-807e-9c584d6afaf1","RayCluster":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:40:52.313Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"0da038df-429a-4a51-807e-9c584d6afaf1","isClusterDeleted":false}
{"level":"info","ts":"2026-11-03T15:40:52.313Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"0da038df-429a-4a51-807e-9c584d6afaf1","deletionMechanism":"ShutdownAfterJobFinishes","RayCluster":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:40:52.313Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-waiting-kjf64","reconcileID":"2924cb28-fc5a-44f0-beb9-0cf9105fab39","cluster":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:40:52.313Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-waiting-kjf64","reconcileID":"2924cb28-fc5a-44f0-beb9-0cf9105fab39","namespace":"test-ns-kbx4w","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-11-03T15:40:52.365Z","logger":"authentication-controller","msg":"Deleted shared ReferenceGrant (last cluster with auth)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-waiting-kjf64","reconcileID":"2924cb28-fc5a-44f0-beb9-0cf9105fab39","name":"kuberay-gateway-access","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:40:52.417Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-waiting-kjf64","reconcileID":"2924cb28-fc5a-44f0-beb9-0cf9105fab39","name":"test-ns-kbx4w-ttl-waiting-kjf64","namespace":"opendatahub"}
{"level":"info","ts":"2026-11-03T15:40:52.469Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-waiting-kjf64","reconcileID":"2924cb28-fc5a-44f0-beb9-0cf9105fab39","configMap":"kube-rbac-proxy-config-ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:40:52.521Z","logger":"authentication-controller","msg":"Deleted service account","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-waiting-kjf64","reconcileID":"2924cb28-fc5a-44f0-beb9-0cf9105fab39","serviceAccount":"ttl-waiting-kjf64-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-11-03T15:40:52.701Z","logger":"raycluster-resource","msg":"default","name":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:40:52.701Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:40:52.875Z","logger":"raycluster-resource","msg":"validate update","name":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:40:52.930Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-waiting-kjf64","reconcileID":"2924cb28-fc5a-44f0-beb9-0cf9105fab39","cluster":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:40:52.930Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-waiting-kjf64","reconcileID":"4eb1ab8b-0835-4878-9321-af0d4e4012be","namespacedName":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:40:52.930Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-waiting-kjf64","reconcileID":"4eb1ab8b-0835-4878-9321-af0d4e4012be","cluster":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:40:52.930Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-waiting-kjf64","reconcileID":"4eb1ab8b-0835-4878-9321-af0d4e4012be","cluster":"ttl-waiting-kjf64"}
{"level":"info","ts":"2026-11-03T15:40:52.930Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-waiting-kjf64","reconcileID":"4eb1ab8b-0835-4878-9321-af0d4e4012be","namespace":"test-ns-kbx4w","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-11-03T15:40:52.930Z","logger":"authentication-controller","msg":"ReferenceGrant already deleted","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-waiting-kjf64","reconcileID":"4eb1ab8b-0835-4878-9321-af0d4e4012be","name":"kuberay-gateway-access"}
{"level":"info","ts":"2026-11-03T15:40:52.930Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"e65ff5d3-02cb-4760-b4b7-0e0d0cb9097b","JobStatus":"","JobDeploymentStatus":"Failed","SubmissionMode":"InteractiveMode"}
{"level":"info","ts":"2026-11-03T15:40:52.930Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"e65ff5d3-02cb-4760-b4b7-0e0d0cb9097b","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Failed","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-11-03 15:40:52 +0000 UTC","Now":"2026-11-03T15:40:52.930Z","ShutdownTime":"2026-11-03T15:40:52.000Z"}
{"level":"info","ts":"2026-11-03T15:40:52.931Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"e65ff5d3-02cb-4760-b4b7-0e0d0cb9097b","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:40:52.931Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"e65ff5d3-02cb-4760-b4b7-0e0d0cb9097b","isClusterDeleted":true}
{"level":"info","ts":"2026-11-03T15:40:52.931Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"e65ff5d3-02cb-4760-b4b7-0e0d0cb9097b","deletionMechanism":"ShutdownAfterJobFinishes","RayCluster":"ttl-waiting-kjf64"}
{"level":"error","ts":"2026-11-03T15:40:52.981Z","logger":"authentication-controller","msg":"Failed to remove finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-waiting-kjf64","reconcileID":"4eb1ab8b-0835-4878-9321-af0d4e4012be","error":"rayclusters.ray.io \"ttl-waiting-kjf64\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleDeletion\n\t/workspace/controllers/ray/authentication_controller.go:195\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:136\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:40:52.981Z","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-waiting-kjf64","reconcileID":"4eb1ab8b-0835-4878-9321-af0d4e4012be"}
{"level":"error","ts":"2026-11-03T15:40:52.981Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-waiting-kjf64","reconcileID":"4eb1ab8b-0835-4878-9321-af0d4e4012be","error":"rayclusters.ray.io \"ttl-waiting-kjf64\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:40:52.981Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-waiting-kjf64","reconcileID":"9e112951-83fe-4080-99b2-00bfc45cb5e5","namespacedName":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:40:52.981Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-waiting-kjf64","reconcileID":"9e112951-83fe-4080-99b2-00bfc45cb5e5"}
{"level":"info","ts":"2026-11-03T15:40:52.982Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-waiting-kjf64","reconcileID":"9e112951-83fe-4080-99b2-00bfc45cb5e5","namespace":"test-ns-kbx4w","authClusters":0}
{"level":"info","ts":"2026-11-03T15:40:52.982Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ttl-waiting-kjf64","namespace":"test-ns-kbx4w"},"namespace":"test-ns-kbx4w","name":"ttl-waiting-kjf64","reconcileID":"9e112951-83fe-4080-99b2-00bfc45cb5e5","namespace":"test-ns-kbx4w"}
{"level":"info","ts":"2026-11-03T15:40:55.961Z","logger":"raycluster-resource","msg":"default","name":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:40:55.961Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"ray-cluster","namespace":"test-ns-4jsqd"}
{"level":"info","ts":"2026-11-03T15:40:56.017Z","logger":"raycluster-resource","msg":"validate create","name":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:40:56.068Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"8b09dd8c-ecb8-43e0-900f-37f24f04cc5c","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:40:56.068Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"namespace":"test-ns-4jsqd","name":"ray-cluster","reconcileID":"fb914563-8552-4ec5-b045-500020ed8dba","namespacedName":{"name":"ray-cluster","namespace":"test-ns-4jsqd"}}
{"level":"info","ts":"2026-11-03T15:40:56.068Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"namespace":"test-ns-4jsqd","name":"ray-cluster","reconcileID":"fb914563-8552-4ec5-b045-500020ed8dba","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:40:56.068Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"namespace":"test-ns-4jsqd","name":"ray-cluster","reconcileID":"fb914563-8552-4ec5-b045-500020ed8dba","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:40:56.068Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"namespace":"test-ns-4jsqd","name":"ray-cluster","reconcileID":"fb914563-8552-4ec5-b045-500020ed8dba","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:40:56.069Z","logger":"controllers.NetworkPolicy","msg":"Reconciling NetworkPolicies for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"506ac71a-0b12-45dc-9302-de7ac421a0be","cluster":"ray-cluster","namespace":"test-ns-4jsqd"}
{"level":"info","ts":"2026-11-03T15:40:56.125Z","logger":"raycluster-resource","msg":"default","name":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:40:56.125Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"ray-cluster","namespace":"test-ns-4jsqd"}
{"level":"info","ts":"2026-11-03T15:40:56.127Z","logger":"controllers.RayCluster","msg":"Created service account for Ray Autoscaler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"e0172a07-f289-42a3-a279-fdfb375bc217","name":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:40:56.128Z","logger":"controllers.NetworkPolicy","msg":"Successfully created NetworkPolicy","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"506ac71a-0b12-45dc-9302-de7ac421a0be","name":"ray-cluster-head"}
{"level":"info","ts":"2026-11-03T15:40:56.183Z","logger":"raycluster-resource","msg":"validate update","name":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:40:56.226Z","logger":"controllers.NetworkPolicy","msg":"Successfully created NetworkPolicy","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"506ac71a-0b12-45dc-9302-de7ac421a0be","name":"ray-cluster-workers-small-group"}
{"level":"info","ts":"2026-11-03T15:40:56.226Z","logger":"controllers.NetworkPolicy","msg":"Successfully reconciled NetworkPolicies for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"506ac71a-0b12-45dc-9302-de7ac421a0be","cluster":"ray-cluster","namespace":"test-ns-4jsqd"}
{"level":"info","ts":"2026-11-03T15:40:56.226Z","logger":"controllers.NetworkPolicy","msg":"Reconciling NetworkPolicies for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b97d9eeb-afc6-4935-a1b0-4ea50732adf6","cluster":"ray-cluster","namespace":"test-ns-4jsqd"}
{"level":"info","ts":"2026-11-03T15:40:56.289Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"namespace":"test-ns-4jsqd","name":"ray-cluster","reconcileID":"fb914563-8552-4ec5-b045-500020ed8dba","name":"ray-cluster-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-11-03T15:40:56.303Z","logger":"controllers.RayCluster","msg":"Created role for Ray Autoscaler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"e0172a07-f289-42a3-a279-fdfb375bc217","name":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:40:56.331Z","logger":"controllers.NetworkPolicy","msg":"Successfully reconciled NetworkPolicies for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b97d9eeb-afc6-4935-a1b0-4ea50732adf6","cluster":"ray-cluster","namespace":"test-ns-4jsqd"}
{"level":"info","ts":"2026-11-03T15:40:56.489Z","logger":"authentication-controller","msg":"Autoscaler RoleBinding does not exist yet, will retry auth SA patch","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"namespace":"test-ns-4jsqd","name":"ray-cluster","reconcileID":"fb914563-8552-4ec5-b045-500020ed8dba","cluster":"ray-cluster"}
{"level":"error","ts":"2026-11-03T15:40:56.559Z","logger":"authentication-controller","msg":"Failed to update AuthenticationReady condition","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"namespace":"test-ns-4jsqd","name":"ray-cluster","reconcileID":"fb914563-8552-4ec5-b045-500020ed8dba","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleOIDCConfiguration\n\t/workspace/controllers/ray/authentication_controller.go:280\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:155\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:40:56.559Z","logger":"authentication-controller","msg":"Autoscaler RoleBinding not ready yet, requeueing","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"namespace":"test-ns-4jsqd","name":"ray-cluster","reconcileID":"fb914563-8552-4ec5-b045-500020ed8dba","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:40:56.559Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"namespace":"test-ns-4jsqd","name":"ray-cluster","reconcileID":"a91b4fef-692e-4ee5-bd74-66cc0616df6e","namespacedName":{"name":"ray-cluster","namespace":"test-ns-4jsqd"}}
{"level":"info","ts":"2026-11-03T15:40:56.559Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"namespace":"test-ns-4jsqd","name":"ray-cluster","reconcileID":"a91b4fef-692e-4ee5-bd74-66cc0616df6e","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:40:56.559Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"namespace":"test-ns-4jsqd","name":"ray-cluster","reconcileID":"a91b4fef-692e-4ee5-bd74-66cc0616df6e","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:40:56.559Z","logger":"authentication-controller","msg":"Autoscaler RoleBinding does not exist yet, will retry auth SA patch","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"namespace":"test-ns-4jsqd","name":"ray-cluster","reconcileID":"a91b4fef-692e-4ee5-bd74-66cc0616df6e","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:40:56.583Z","logger":"controllers.RayCluster","msg":"Created role binding for Ray Autoscaler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"e0172a07-f289-42a3-a279-fdfb375bc217","name":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:40:56.583Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"e0172a07-f289-42a3-a279-fdfb375bc217"}
{"level":"info","ts":"2026-11-03T15:40:56.583Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"e0172a07-f289-42a3-a279-fdfb375bc217"}
{"level":"info","ts":"2026-11-03T15:40:56.617Z","logger":"authentication-controller","msg":"Autoscaler RoleBinding not ready yet, requeueing","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"namespace":"test-ns-4jsqd","name":"ray-cluster","reconcileID":"a91b4fef-692e-4ee5-bd74-66cc0616df6e","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:40:56.638Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"namespace":"test-ns-4jsqd","name":"ray-cluster","reconcileID":"fbc93e6b-f757-432b-a4a2-6e8ea0b11c35","namespacedName":{"name":"ray-cluster","namespace":"test-ns-4jsqd"}}
{"level":"info","ts":"2026-11-03T15:40:56.638Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"namespace":"test-ns-4jsqd","name":"ray-cluster","reconcileID":"fbc93e6b-f757-432b-a4a2-6e8ea0b11c35","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:40:56.638Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"namespace":"test-ns-4jsqd","name":"ray-cluster","reconcileID":"fbc93e6b-f757-432b-a4a2-6e8ea0b11c35","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:40:56.638Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"e0172a07-f289-42a3-a279-fdfb375bc217","name":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:40:56.638Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"e0172a07-f289-42a3-a279-fdfb375bc217"}
{"level":"info","ts":"2026-11-03T15:40:56.687Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"e0172a07-f289-42a3-a279-fdfb375bc217","error":"certificate ray-head-cert-ray-cluster not found: certificates.cert-manager.io \"ray-head-cert-ray-cluster\" not found"}
{"level":"error","ts":"2026-11-03T15:40:56.687Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"e0172a07-f289-42a3-a279-fdfb375bc217","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster not found: certificates.cert-manager.io \"ray-head-cert-ray-cluster\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:40:56.687Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"e0172a07-f289-42a3-a279-fdfb375bc217","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster not found: certificates.cert-manager.io \"ray-head-cert-ray-cluster\" not found"}
{"level":"info","ts":"2026-11-03T15:40:56.687Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"e0172a07-f289-42a3-a279-fdfb375bc217","filter labels":["test-ns-4jsqd",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:40:56.687Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"e0172a07-f289-42a3-a279-fdfb375bc217","filter labels":["test-ns-4jsqd",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:40:56.687Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"e0172a07-f289-42a3-a279-fdfb375bc217","filter labels":["test-ns-4jsqd",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:40:56.687Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"e0172a07-f289-42a3-a279-fdfb375bc217","name":"ray-cluster","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:40:56Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:40:56.712Z","logger":"authentication-controller","msg":"Added auth proxy SA to autoscaler RoleBinding","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"namespace":"test-ns-4jsqd","name":"ray-cluster","reconcileID":"fbc93e6b-f757-432b-a4a2-6e8ea0b11c35","roleBinding":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"authServiceAccount":"ray-cluster-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-11-03T15:40:56.741Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"e0172a07-f289-42a3-a279-fdfb375bc217","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-4jsqd","name":"ray-cluster"}}
{"level":"info","ts":"2026-11-03T15:40:56.741Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"e0172a07-f289-42a3-a279-fdfb375bc217"}
{"level":"error","ts":"2026-11-03T15:40:56.741Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"e0172a07-f289-42a3-a279-fdfb375bc217","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster not found: certificates.cert-manager.io \"ray-head-cert-ray-cluster\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:40:56.741Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"2192fa95-9595-4b4a-b9c9-9c89d46367e5"}
{"level":"info","ts":"2026-11-03T15:40:56.741Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"2192fa95-9595-4b4a-b9c9-9c89d46367e5"}
{"level":"info","ts":"2026-11-03T15:40:56.741Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"2192fa95-9595-4b4a-b9c9-9c89d46367e5","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:40:56.741Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"2192fa95-9595-4b4a-b9c9-9c89d46367e5"}
{"level":"info","ts":"2026-11-03T15:40:56.764Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"namespace":"test-ns-4jsqd","name":"ray-cluster","reconcileID":"fbc93e6b-f757-432b-a4a2-6e8ea0b11c35","name":"kuberay-gateway-access","namespace":"test-ns-4jsqd","operation":"created"}
{"level":"info","ts":"2026-11-03T15:40:56.790Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"2192fa95-9595-4b4a-b9c9-9c89d46367e5","error":"certificate ray-head-cert-ray-cluster not found: certificates.cert-manager.io \"ray-head-cert-ray-cluster\" not found"}
{"level":"error","ts":"2026-11-03T15:40:56.790Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"2192fa95-9595-4b4a-b9c9-9c89d46367e5","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster not found: certificates.cert-manager.io \"ray-head-cert-ray-cluster\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:40:56.790Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"2192fa95-9595-4b4a-b9c9-9c89d46367e5","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster not found: certificates.cert-manager.io \"ray-head-cert-ray-cluster\" not found"}
{"level":"info","ts":"2026-11-03T15:40:56.790Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"2192fa95-9595-4b4a-b9c9-9c89d46367e5","filter labels":["test-ns-4jsqd",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:40:56.790Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"2192fa95-9595-4b4a-b9c9-9c89d46367e5","filter labels":["test-ns-4jsqd",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:40:56.790Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"2192fa95-9595-4b4a-b9c9-9c89d46367e5","filter labels":["test-ns-4jsqd",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:40:56.790Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"2192fa95-9595-4b4a-b9c9-9c89d46367e5","name":"ray-cluster","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{},"conditions":[{"type":"AuthenticationReady","status":"False","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:40:56Z","reason":"AuthenticationFailed","message":"Failed to create authentication resources: autoscaler RoleBinding not found yet"}]},"new status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:40:56Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"False","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:40:56Z","reason":"AuthenticationFailed","message":"Failed to create authentication resources: autoscaler RoleBinding not found yet"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:40:56.820Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"namespace":"test-ns-4jsqd","name":"ray-cluster","reconcileID":"fbc93e6b-f757-432b-a4a2-6e8ea0b11c35","name":"test-ns-4jsqd-ray-cluster","operation":"created"}
{"level":"info","ts":"2026-11-03T15:40:56.851Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"2192fa95-9595-4b4a-b9c9-9c89d46367e5"}
{"level":"error","ts":"2026-11-03T15:40:56.851Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"2192fa95-9595-4b4a-b9c9-9c89d46367e5","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster not found: certificates.cert-manager.io \"ray-head-cert-ray-cluster\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:40:56.861Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"8cce1642-13b3-469d-a280-752783f0d84a"}
{"level":"info","ts":"2026-11-03T15:40:56.861Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"8cce1642-13b3-469d-a280-752783f0d84a"}
{"level":"info","ts":"2026-11-03T15:40:56.861Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"8cce1642-13b3-469d-a280-752783f0d84a","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:40:56.861Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"8cce1642-13b3-469d-a280-752783f0d84a"}
{"level":"info","ts":"2026-11-03T15:40:56.873Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"namespace":"test-ns-4jsqd","name":"ray-cluster","reconcileID":"fbc93e6b-f757-432b-a4a2-6e8ea0b11c35","name":"kube-rbac-proxy-config-ray-cluster","operation":"created"}
{"level":"info","ts":"2026-11-03T15:40:56.911Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"8cce1642-13b3-469d-a280-752783f0d84a","error":"certificate ray-head-cert-ray-cluster is not ready at current generation 1"}
{"level":"error","ts":"2026-11-03T15:40:56.911Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"8cce1642-13b3-469d-a280-752783f0d84a","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 1","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:40:56.911Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"8cce1642-13b3-469d-a280-752783f0d84a","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 1"}
{"level":"info","ts":"2026-11-03T15:40:56.911Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"8cce1642-13b3-469d-a280-752783f0d84a","filter labels":["test-ns-4jsqd",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:40:56.911Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"8cce1642-13b3-469d-a280-752783f0d84a","filter labels":["test-ns-4jsqd",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:40:56.911Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"8cce1642-13b3-469d-a280-752783f0d84a","filter labels":["test-ns-4jsqd",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:40:56.911Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"8cce1642-13b3-469d-a280-752783f0d84a"}
{"level":"error","ts":"2026-11-03T15:40:56.911Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"8cce1642-13b3-469d-a280-752783f0d84a","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-03T15:40:56.927Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"namespace":"test-ns-4jsqd","name":"ray-cluster","reconcileID":"fbc93e6b-f757-432b-a4a2-6e8ea0b11c35","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:167\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:40:56.927Z","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"namespace":"test-ns-4jsqd","name":"ray-cluster","reconcileID":"fbc93e6b-f757-432b-a4a2-6e8ea0b11c35"}
{"level":"error","ts":"2026-11-03T15:40:56.927Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"namespace":"test-ns-4jsqd","name":"ray-cluster","reconcileID":"fbc93e6b-f757-432b-a4a2-6e8ea0b11c35","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:40:56.932Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"namespace":"test-ns-4jsqd","name":"ray-cluster","reconcileID":"88f0c06f-2edf-4b4d-8bdb-b51e20d33ef2","namespacedName":{"name":"ray-cluster","namespace":"test-ns-4jsqd"}}
{"level":"info","ts":"2026-11-03T15:40:56.932Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"namespace":"test-ns-4jsqd","name":"ray-cluster","reconcileID":"88f0c06f-2edf-4b4d-8bdb-b51e20d33ef2","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:40:56.932Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"namespace":"test-ns-4jsqd","name":"ray-cluster","reconcileID":"88f0c06f-2edf-4b4d-8bdb-b51e20d33ef2","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:40:56.932Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"5195910d-893a-4cd7-8fe1-1bd64afbbb0f"}
{"level":"info","ts":"2026-11-03T15:40:56.932Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"5195910d-893a-4cd7-8fe1-1bd64afbbb0f"}
{"level":"info","ts":"2026-11-03T15:40:56.932Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"5195910d-893a-4cd7-8fe1-1bd64afbbb0f","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:40:56.933Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"5195910d-893a-4cd7-8fe1-1bd64afbbb0f"}
{"level":"info","ts":"2026-11-03T15:40:56.982Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"5195910d-893a-4cd7-8fe1-1bd64afbbb0f","error":"certificate ray-head-cert-ray-cluster is not ready at current generation 1"}
{"level":"error","ts":"2026-11-03T15:40:56.982Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"5195910d-893a-4cd7-8fe1-1bd64afbbb0f","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 1","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:40:56.982Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"5195910d-893a-4cd7-8fe1-1bd64afbbb0f","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 1"}
{"level":"info","ts":"2026-11-03T15:40:56.982Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"5195910d-893a-4cd7-8fe1-1bd64afbbb0f","filter labels":["test-ns-4jsqd",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:40:56.982Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"5195910d-893a-4cd7-8fe1-1bd64afbbb0f","filter labels":["test-ns-4jsqd",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:40:56.983Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"5195910d-893a-4cd7-8fe1-1bd64afbbb0f","filter labels":["test-ns-4jsqd",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:40:56.983Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"5195910d-893a-4cd7-8fe1-1bd64afbbb0f"}
{"level":"error","ts":"2026-11-03T15:40:56.983Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"5195910d-893a-4cd7-8fe1-1bd64afbbb0f","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:40:56.994Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"namespace":"test-ns-4jsqd","name":"ray-cluster","reconcileID":"88f0c06f-2edf-4b4d-8bdb-b51e20d33ef2","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:40:57.023Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"83fb095b-f462-4ae1-b609-24d3dd778da3"}
{"level":"info","ts":"2026-11-03T15:40:57.023Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"83fb095b-f462-4ae1-b609-24d3dd778da3"}
{"level":"info","ts":"2026-11-03T15:40:57.023Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"83fb095b-f462-4ae1-b609-24d3dd778da3","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:40:57.023Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"83fb095b-f462-4ae1-b609-24d3dd778da3"}
{"level":"info","ts":"2026-11-03T15:40:57.029Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"8b09dd8c-ecb8-43e0-900f-37f24f04cc5c","certificate":"ray-head-cert-ray-cluster"}
{"level":"info","ts":"2026-11-03T15:40:57.029Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"8b09dd8c-ecb8-43e0-900f-37f24f04cc5c"}
{"level":"info","ts":"2026-11-03T15:40:57.029Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"a08c1081-5757-402a-b15b-d1320468012d","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:40:57.073Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"83fb095b-f462-4ae1-b609-24d3dd778da3","error":"certificate ray-head-cert-ray-cluster is not ready at current generation 1"}
{"level":"error","ts":"2026-11-03T15:40:57.073Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"83fb095b-f462-4ae1-b609-24d3dd778da3","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 1","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:40:57.073Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"83fb095b-f462-4ae1-b609-24d3dd778da3","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 1"}
{"level":"info","ts":"2026-11-03T15:40:57.073Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"83fb095b-f462-4ae1-b609-24d3dd778da3","filter labels":["test-ns-4jsqd",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:40:57.073Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"83fb095b-f462-4ae1-b609-24d3dd778da3","filter labels":["test-ns-4jsqd",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:40:57.073Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"83fb095b-f462-4ae1-b609-24d3dd778da3","filter labels":["test-ns-4jsqd",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:40:57.073Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"83fb095b-f462-4ae1-b609-24d3dd778da3"}
{"level":"error","ts":"2026-11-03T15:40:57.073Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"83fb095b-f462-4ae1-b609-24d3dd778da3","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:40:57.153Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"3ad53523-f555-4af1-b595-588844a9e4ab"}
{"level":"info","ts":"2026-11-03T15:40:57.153Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"3ad53523-f555-4af1-b595-588844a9e4ab"}
{"level":"info","ts":"2026-11-03T15:40:57.153Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"3ad53523-f555-4af1-b595-588844a9e4ab","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:40:57.153Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"3ad53523-f555-4af1-b595-588844a9e4ab"}
{"level":"info","ts":"2026-11-03T15:40:57.203Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"3ad53523-f555-4af1-b595-588844a9e4ab","error":"certificate ray-head-cert-ray-cluster is not ready at current generation 1"}
{"level":"error","ts":"2026-11-03T15:40:57.203Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"3ad53523-f555-4af1-b595-588844a9e4ab","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 1","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:40:57.203Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"3ad53523-f555-4af1-b595-588844a9e4ab","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 1"}
{"level":"info","ts":"2026-11-03T15:40:57.204Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"3ad53523-f555-4af1-b595-588844a9e4ab","filter labels":["test-ns-4jsqd",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:40:57.204Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"3ad53523-f555-4af1-b595-588844a9e4ab","filter labels":["test-ns-4jsqd",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:40:57.204Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"3ad53523-f555-4af1-b595-588844a9e4ab","filter labels":["test-ns-4jsqd",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:40:57.204Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"3ad53523-f555-4af1-b595-588844a9e4ab"}
{"level":"error","ts":"2026-11-03T15:40:57.204Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"3ad53523-f555-4af1-b595-588844a9e4ab","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:40:57.365Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"23484697-1414-49a2-be8a-d58ae42e0375"}
{"level":"info","ts":"2026-11-03T15:40:57.365Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"23484697-1414-49a2-be8a-d58ae42e0375"}
{"level":"info","ts":"2026-11-03T15:40:57.365Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"23484697-1414-49a2-be8a-d58ae42e0375","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:40:57.365Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"23484697-1414-49a2-be8a-d58ae42e0375"}
{"level":"info","ts":"2026-11-03T15:40:57.415Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"23484697-1414-49a2-be8a-d58ae42e0375","error":"certificate ray-head-cert-ray-cluster is not ready at current generation 1"}
{"level":"error","ts":"2026-11-03T15:40:57.415Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"23484697-1414-49a2-be8a-d58ae42e0375","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 1","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:40:57.415Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"23484697-1414-49a2-be8a-d58ae42e0375","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 1"}
{"level":"info","ts":"2026-11-03T15:40:57.415Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"23484697-1414-49a2-be8a-d58ae42e0375","filter labels":["test-ns-4jsqd",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:40:57.415Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"23484697-1414-49a2-be8a-d58ae42e0375","filter labels":["test-ns-4jsqd",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:40:57.415Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"23484697-1414-49a2-be8a-d58ae42e0375","filter labels":["test-ns-4jsqd",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:40:57.415Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"23484697-1414-49a2-be8a-d58ae42e0375"}
{"level":"error","ts":"2026-11-03T15:40:57.415Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"23484697-1414-49a2-be8a-d58ae42e0375","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:40:57.479Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"a08c1081-5757-402a-b15b-d1320468012d","certificate":"ray-head-cert-ray-cluster"}
{"level":"info","ts":"2026-11-03T15:40:57.479Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"a08c1081-5757-402a-b15b-d1320468012d"}
{"level":"info","ts":"2026-11-03T15:40:57.735Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"d64fc28a-466f-4d26-bd6c-e06233bdd4db"}
{"level":"info","ts":"2026-11-03T15:40:57.735Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"d64fc28a-466f-4d26-bd6c-e06233bdd4db"}
{"level":"info","ts":"2026-11-03T15:40:57.735Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"d64fc28a-466f-4d26-bd6c-e06233bdd4db","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:40:57.736Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"d64fc28a-466f-4d26-bd6c-e06233bdd4db"}
{"level":"info","ts":"2026-11-03T15:40:57.786Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"d64fc28a-466f-4d26-bd6c-e06233bdd4db","error":"certificate ray-head-cert-ray-cluster is not ready at current generation 1"}
{"level":"error","ts":"2026-11-03T15:40:57.786Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"d64fc28a-466f-4d26-bd6c-e06233bdd4db","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 1","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:40:57.786Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"d64fc28a-466f-4d26-bd6c-e06233bdd4db","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 1"}
{"level":"info","ts":"2026-11-03T15:40:57.786Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"d64fc28a-466f-4d26-bd6c-e06233bdd4db","filter labels":["test-ns-4jsqd",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:40:57.786Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"d64fc28a-466f-4d26-bd6c-e06233bdd4db","filter labels":["test-ns-4jsqd",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:40:57.786Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"d64fc28a-466f-4d26-bd6c-e06233bdd4db","filter labels":["test-ns-4jsqd",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:40:57.786Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"d64fc28a-466f-4d26-bd6c-e06233bdd4db"}
{"level":"error","ts":"2026-11-03T15:40:57.786Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"d64fc28a-466f-4d26-bd6c-e06233bdd4db","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:40:58.426Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"88c3bd5e-b413-41ed-9b2a-29bd320efe51"}
{"level":"info","ts":"2026-11-03T15:40:58.426Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"88c3bd5e-b413-41ed-9b2a-29bd320efe51"}
{"level":"info","ts":"2026-11-03T15:40:58.426Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"88c3bd5e-b413-41ed-9b2a-29bd320efe51","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:40:58.427Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"88c3bd5e-b413-41ed-9b2a-29bd320efe51"}
{"level":"info","ts":"2026-11-03T15:40:58.477Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"88c3bd5e-b413-41ed-9b2a-29bd320efe51","error":"certificate ray-head-cert-ray-cluster is not ready at current generation 1"}
{"level":"error","ts":"2026-11-03T15:40:58.477Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"88c3bd5e-b413-41ed-9b2a-29bd320efe51","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 1","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:40:58.477Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"88c3bd5e-b413-41ed-9b2a-29bd320efe51","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 1"}
{"level":"info","ts":"2026-11-03T15:40:58.477Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"88c3bd5e-b413-41ed-9b2a-29bd320efe51","filter labels":["test-ns-4jsqd",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:40:58.477Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"88c3bd5e-b413-41ed-9b2a-29bd320efe51","filter labels":["test-ns-4jsqd",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:40:58.477Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"88c3bd5e-b413-41ed-9b2a-29bd320efe51","filter labels":["test-ns-4jsqd",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:40:58.477Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"88c3bd5e-b413-41ed-9b2a-29bd320efe51"}
{"level":"error","ts":"2026-11-03T15:40:58.477Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"88c3bd5e-b413-41ed-9b2a-29bd320efe51","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:40:59.758Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"a9999a04-b6c0-4855-873e-a2e9213f2890"}
{"level":"info","ts":"2026-11-03T15:40:59.758Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"a9999a04-b6c0-4855-873e-a2e9213f2890"}
{"level":"info","ts":"2026-11-03T15:40:59.758Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"a9999a04-b6c0-4855-873e-a2e9213f2890","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:40:59.758Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"a9999a04-b6c0-4855-873e-a2e9213f2890"}
{"level":"info","ts":"2026-11-03T15:40:59.861Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"a9999a04-b6c0-4855-873e-a2e9213f2890","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:40:59.862Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"a9999a04-b6c0-4855-873e-a2e9213f2890","cluster":"ray-cluster","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-11-03T15:40:59.862Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"a9999a04-b6c0-4855-873e-a2e9213f2890","cluster":"ray-cluster","authType":"IntegratedOAuth","serviceAccount":"ray-cluster-oauth-proxy-sa","containerCount":3}
{"level":"info","ts":"2026-11-03T15:40:59.862Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"a9999a04-b6c0-4855-873e-a2e9213f2890","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"ray-cluster","ray.io/group":"headgroup","ray.io/identifier":"ray-cluster-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-11-03T15:40:59.862Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"a9999a04-b6c0-4855-873e-a2e9213f2890","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","no-monitor":"true","num-cpus":"0"},"Ray container resource":{"limits":{"cpu":"1","memory":"4G"},"requests":{"cpu":"1","memory":"2G"}}}
{"level":"info","ts":"2026-11-03T15:40:59.862Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"a9999a04-b6c0-4855-873e-a2e9213f2890","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --num-cpus=0 "}
{"level":"info","ts":"2026-11-03T15:40:59.862Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"a9999a04-b6c0-4855-873e-a2e9213f2890","rayNodeType":"head","generatedCmd":"ulimit -n ${RAY_START_ULIMIT_OPEN_FILES:-65536}; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --num-cpus=0 "}
{"level":"info","ts":"2026-11-03T15:40:59.862Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"a9999a04-b6c0-4855-873e-a2e9213f2890","enabled":true}
{"level":"info","ts":"2026-11-03T15:40:59.984Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"a9999a04-b6c0-4855-873e-a2e9213f2890","name":"ray-cluster-head-9lzd6"}
{"level":"info","ts":"2026-11-03T15:40:59.984Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"a9999a04-b6c0-4855-873e-a2e9213f2890","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-11-03T15:40:59.984Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"a9999a04-b6c0-4855-873e-a2e9213f2890","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:40:59.984Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"a9999a04-b6c0-4855-873e-a2e9213f2890","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-11-03T15:40:59.984Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"a9999a04-b6c0-4855-873e-a2e9213f2890","group":"small-group","indices":{}}
{"level":"info","ts":"2026-11-03T15:40:59.984Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"a9999a04-b6c0-4855-873e-a2e9213f2890","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:40:59.985Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"a9999a04-b6c0-4855-873e-a2e9213f2890","podName":"ray-cluster-head-9lzd6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:40:59.985Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"a9999a04-b6c0-4855-873e-a2e9213f2890","podName":"ray-cluster-head-9lzd6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:40:59.985Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"a9999a04-b6c0-4855-873e-a2e9213f2890","name":"ray-cluster","old status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:40:56Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:40:56Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:40:59Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ray-cluster-head-9lzd6","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:40:56Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:41:00.067Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"780ffacf-5348-4bb3-aaae-2c2ff1cefef5","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:41:00.067Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"4eb1c41a-9289-468b-b44e-5eccf62ee63f"}
{"level":"info","ts":"2026-11-03T15:41:00.067Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"4eb1c41a-9289-468b-b44e-5eccf62ee63f"}
{"level":"info","ts":"2026-11-03T15:41:00.067Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"4eb1c41a-9289-468b-b44e-5eccf62ee63f","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:41:00.067Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"4eb1c41a-9289-468b-b44e-5eccf62ee63f","Found 1 head Pod":"ray-cluster-head-9lzd6","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:41:00.068Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"4eb1c41a-9289-468b-b44e-5eccf62ee63f","head Pod":"ray-cluster-head-9lzd6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-9lzd6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:41:00.068Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"4eb1c41a-9289-468b-b44e-5eccf62ee63f","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-11-03T15:41:00.068Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"4eb1c41a-9289-468b-b44e-5eccf62ee63f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:41:00.068Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"4eb1c41a-9289-468b-b44e-5eccf62ee63f","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-11-03T15:41:00.068Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"4eb1c41a-9289-468b-b44e-5eccf62ee63f","group":"small-group","indices":{}}
{"level":"info","ts":"2026-11-03T15:41:00.068Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"4eb1c41a-9289-468b-b44e-5eccf62ee63f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:41:00.068Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"4eb1c41a-9289-468b-b44e-5eccf62ee63f","podName":"ray-cluster-head-9lzd6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:41:00.068Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"4eb1c41a-9289-468b-b44e-5eccf62ee63f","podName":"ray-cluster-head-9lzd6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:41:00.068Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"4eb1c41a-9289-468b-b44e-5eccf62ee63f","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:41:00.068Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"4eb1c41a-9289-468b-b44e-5eccf62ee63f","seconds":300}
{"level":"info","ts":"2026-11-03T15:41:00.075Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"9e3da86d-1285-4dda-824a-f2d82e4c7dc3"}
{"level":"info","ts":"2026-11-03T15:41:00.075Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"9e3da86d-1285-4dda-824a-f2d82e4c7dc3"}
{"level":"info","ts":"2026-11-03T15:41:00.075Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"9e3da86d-1285-4dda-824a-f2d82e4c7dc3","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:41:00.075Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"9e3da86d-1285-4dda-824a-f2d82e4c7dc3","Found 1 head Pod":"ray-cluster-head-9lzd6","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:41:00.075Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"9e3da86d-1285-4dda-824a-f2d82e4c7dc3","head Pod":"ray-cluster-head-9lzd6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-9lzd6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:41:00.075Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"9e3da86d-1285-4dda-824a-f2d82e4c7dc3","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-11-03T15:41:00.075Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"9e3da86d-1285-4dda-824a-f2d82e4c7dc3","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:41:00.075Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"9e3da86d-1285-4dda-824a-f2d82e4c7dc3","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-11-03T15:41:00.075Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"9e3da86d-1285-4dda-824a-f2d82e4c7dc3","group":"small-group","indices":{}}
{"level":"info","ts":"2026-11-03T15:41:00.075Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"9e3da86d-1285-4dda-824a-f2d82e4c7dc3","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:41:00.075Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"9e3da86d-1285-4dda-824a-f2d82e4c7dc3","podName":"ray-cluster-head-9lzd6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:41:00.075Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"9e3da86d-1285-4dda-824a-f2d82e4c7dc3","podName":"ray-cluster-head-9lzd6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:41:00.075Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"9e3da86d-1285-4dda-824a-f2d82e4c7dc3","name":"ray-cluster","old status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:40:59Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ray-cluster-head-9lzd6","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:40:56Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:41:00Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ray-cluster-head-9lzd6","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:40:56Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"PodInitializing","message":"containers with unready status: [ray-head autoscaler kube-rbac-proxy]; autoscaler: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:41:00.888Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"4e3c04d4-b063-44f5-8254-c0a4aac438d1","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:41:01.058Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"996e8724-a66e-46f8-b343-6324fceef7ef"}
{"level":"info","ts":"2026-11-03T15:41:01.058Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"996e8724-a66e-46f8-b343-6324fceef7ef"}
{"level":"info","ts":"2026-11-03T15:41:01.058Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"996e8724-a66e-46f8-b343-6324fceef7ef","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:41:01.058Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"996e8724-a66e-46f8-b343-6324fceef7ef","Found 1 head Pod":"ray-cluster-head-9lzd6","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:41:01.058Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"996e8724-a66e-46f8-b343-6324fceef7ef","head Pod":"ray-cluster-head-9lzd6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-9lzd6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:41:01.058Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"996e8724-a66e-46f8-b343-6324fceef7ef","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-11-03T15:41:01.058Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"996e8724-a66e-46f8-b343-6324fceef7ef","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:41:01.058Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"996e8724-a66e-46f8-b343-6324fceef7ef","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-11-03T15:41:01.058Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"996e8724-a66e-46f8-b343-6324fceef7ef","group":"small-group","indices":{}}
{"level":"info","ts":"2026-11-03T15:41:01.058Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"996e8724-a66e-46f8-b343-6324fceef7ef","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:41:01.058Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"996e8724-a66e-46f8-b343-6324fceef7ef","podName":"ray-cluster-head-9lzd6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:41:01.058Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"996e8724-a66e-46f8-b343-6324fceef7ef","podName":"ray-cluster-head-9lzd6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:41:01.058Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"996e8724-a66e-46f8-b343-6324fceef7ef","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:41:01.058Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"996e8724-a66e-46f8-b343-6324fceef7ef","seconds":300}
{"level":"info","ts":"2026-11-03T15:41:01.588Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"d15e3d1b-6cd0-4d1d-ace0-dd62490d3878"}
{"level":"info","ts":"2026-11-03T15:41:01.588Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"d15e3d1b-6cd0-4d1d-ace0-dd62490d3878"}
{"level":"info","ts":"2026-11-03T15:41:01.588Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"d15e3d1b-6cd0-4d1d-ace0-dd62490d3878","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:41:01.588Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"d15e3d1b-6cd0-4d1d-ace0-dd62490d3878","Found 1 head Pod":"ray-cluster-head-9lzd6","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:41:01.588Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"d15e3d1b-6cd0-4d1d-ace0-dd62490d3878","head Pod":"ray-cluster-head-9lzd6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-9lzd6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:41:01.588Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"d15e3d1b-6cd0-4d1d-ace0-dd62490d3878","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-11-03T15:41:01.588Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"d15e3d1b-6cd0-4d1d-ace0-dd62490d3878","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:41:01.588Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"d15e3d1b-6cd0-4d1d-ace0-dd62490d3878","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-11-03T15:41:01.588Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"d15e3d1b-6cd0-4d1d-ace0-dd62490d3878","group":"small-group","indices":{}}
{"level":"info","ts":"2026-11-03T15:41:01.588Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"d15e3d1b-6cd0-4d1d-ace0-dd62490d3878","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:41:01.588Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"d15e3d1b-6cd0-4d1d-ace0-dd62490d3878","podName":"ray-cluster-head-9lzd6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:41:01.588Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"d15e3d1b-6cd0-4d1d-ace0-dd62490d3878","podName":"ray-cluster-head-9lzd6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:41:01.588Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"d15e3d1b-6cd0-4d1d-ace0-dd62490d3878","name":"ray-cluster","old status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:41:00Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ray-cluster-head-9lzd6","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:40:56Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"PodInitializing","message":"containers with unready status: [ray-head autoscaler kube-rbac-proxy]; autoscaler: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:41:01Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.56","serviceIP":"10.134.0.56","podName":"ray-cluster-head-9lzd6","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:40:56Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"PodInitializing","message":"containers with unready status: [ray-head autoscaler kube-rbac-proxy]; autoscaler: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:41:01.695Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"55200f65-d297-4c55-a87c-3afbdf9927e6","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:41:01.998Z","logger":"controllers.RayClusterMTLS","msg":"Updating head certificate","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"55200f65-d297-4c55-a87c-3afbdf9927e6","certificate":"ray-head-cert-ray-cluster","ipAddresses":["10.134.0.56","127.0.0.1"]}
{"level":"info","ts":"2026-11-03T15:41:02.410Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"55200f65-d297-4c55-a87c-3afbdf9927e6","certificate":"ray-head-cert-ray-cluster"}
{"level":"info","ts":"2026-11-03T15:41:02.410Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"55200f65-d297-4c55-a87c-3afbdf9927e6"}
{"level":"info","ts":"2026-11-03T15:41:03.649Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"896f958d-b5f2-4dbe-b2cb-20b8a7831cb2"}
{"level":"info","ts":"2026-11-03T15:41:03.649Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"896f958d-b5f2-4dbe-b2cb-20b8a7831cb2"}
{"level":"info","ts":"2026-11-03T15:41:03.649Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"896f958d-b5f2-4dbe-b2cb-20b8a7831cb2","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:41:03.650Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"896f958d-b5f2-4dbe-b2cb-20b8a7831cb2","Found 1 head Pod":"ray-cluster-head-9lzd6","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:41:03.650Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"896f958d-b5f2-4dbe-b2cb-20b8a7831cb2","head Pod":"ray-cluster-head-9lzd6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-9lzd6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:41:03.650Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"896f958d-b5f2-4dbe-b2cb-20b8a7831cb2","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-11-03T15:41:03.650Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"896f958d-b5f2-4dbe-b2cb-20b8a7831cb2","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:41:03.650Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"896f958d-b5f2-4dbe-b2cb-20b8a7831cb2","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-11-03T15:41:03.650Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"896f958d-b5f2-4dbe-b2cb-20b8a7831cb2","group":"small-group","indices":{}}
{"level":"info","ts":"2026-11-03T15:41:03.650Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"896f958d-b5f2-4dbe-b2cb-20b8a7831cb2","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:41:03.650Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"896f958d-b5f2-4dbe-b2cb-20b8a7831cb2","podName":"ray-cluster-head-9lzd6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:41:03.650Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"896f958d-b5f2-4dbe-b2cb-20b8a7831cb2","podName":"ray-cluster-head-9lzd6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:41:03.650Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"896f958d-b5f2-4dbe-b2cb-20b8a7831cb2","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:41:03.650Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"896f958d-b5f2-4dbe-b2cb-20b8a7831cb2","seconds":300}
{"level":"info","ts":"2026-11-03T15:41:03.780Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"e85552dc-48f6-412b-a1a6-de4e1f99cb24","JobStatus":"SUCCEEDED","JobDeploymentStatus":"Complete","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:41:03.780Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"e85552dc-48f6-412b-a1a6-de4e1f99cb24","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Complete","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-11-03 15:38:49 +0000 UTC","Now":"2026-11-03T15:41:03.780Z","ShutdownTime":"2026-11-03T15:38:49.000Z"}
{"level":"info","ts":"2026-11-03T15:41:03.780Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"e85552dc-48f6-412b-a1a6-de4e1f99cb24","RayCluster":{"name":"counter-auth-qmw8r","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:41:03.780Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"e85552dc-48f6-412b-a1a6-de4e1f99cb24","isClusterDeleted":true}
{"level":"info","ts":"2026-11-03T15:41:03.780Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"e85552dc-48f6-412b-a1a6-de4e1f99cb24","deletionMechanism":"ShutdownAfterJobFinishes","RayCluster":"counter-auth-qmw8r"}
{"level":"info","ts":"2026-11-03T15:41:03.787Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"a91d1443-bd4e-447c-9bd6-74b8774facfa","JobStatus":"FAILED","JobDeploymentStatus":"Failed","SubmissionMode":"K8sJobMode"}
{"level":"info","ts":"2026-11-03T15:41:03.787Z","logger":"controllers.RayJob","msg":"Evaluating job deletion policy based on ShutdownAfterJobFinishes","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"a91d1443-bd4e-447c-9bd6-74b8774facfa","deletionMechanism":"ShutdownAfterJobFinishes","JobDeploymentStatus":"Failed","ShutdownAfterJobFinishes":true,"ClusterSelector":null,"ttlSecondsAfterFinished":0,"Status.endTime":"2026-11-03 15:34:23 +0000 UTC","Now":"2026-11-03T15:41:03.787Z","ShutdownTime":"2026-11-03T15:34:23.000Z"}
{"level":"info","ts":"2026-11-03T15:41:03.787Z","logger":"controllers.RayJob","msg":"The associated RayCluster for RayJob has been already deleted and it can not be found","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"a91d1443-bd4e-447c-9bd6-74b8774facfa","RayCluster":{"name":"submitter-timeout-g4hph","namespace":"test-ns-kbx4w"}}
{"level":"info","ts":"2026-11-03T15:41:03.787Z","logger":"controllers.RayJob","msg":"deleteClusterResources","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"a91d1443-bd4e-447c-9bd6-74b8774facfa","isClusterDeleted":true}
{"level":"info","ts":"2026-11-03T15:41:03.787Z","logger":"controllers.RayJob","msg":"RayCluster is deleted","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"a91d1443-bd4e-447c-9bd6-74b8774facfa","deletionMechanism":"ShutdownAfterJobFinishes","RayCluster":"submitter-timeout-g4hph"}
{"level":"info","ts":"2026-11-03T15:41:03.980Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","namespace":"test-ns-kbx4w"},"reconcileID":"77022209-600a-4681-a960-bf4fb4646ade"}
{"level":"info","ts":"2026-11-03T15:41:03.989Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"95ba5042-9a02-4310-93c6-d046669a7d28","DeletionTimestamp":"2026-11-03 15:41:03 +0000 UTC"}
{"level":"info","ts":"2026-11-03T15:41:03.989Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"95ba5042-9a02-4310-93c6-d046669a7d28","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-11-03T15:41:04.050Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"invalid-yamlstr","namespace":"test-ns-kbx4w"},"reconcileID":"7042a0cd-7ac7-4d2b-8e52-608c9932c584"}
{"level":"info","ts":"2026-11-03T15:41:04.050Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"943e00fc-9643-499a-9387-35c49517354d","DeletionTimestamp":"2026-11-03 15:41:03 +0000 UTC"}
{"level":"error","ts":"2026-11-03T15:41:04.050Z","logger":"controllers.RayJob","msg":"Failed to get RayCluster","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"943e00fc-9643-499a-9387-35c49517354d","error":"RayCluster.ray.io \"long-running-hvml9\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:120\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-03T15:41:04.050Z","logger":"controllers.RayJob","msg":"Failed to stop job for RayJob","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"943e00fc-9643-499a-9387-35c49517354d","error":"Post \"http:///api/jobs/long-running-cdx2h/stop\": http: no Host in request URL","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:128\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:41:04.050Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"943e00fc-9643-499a-9387-35c49517354d","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-11-03T15:41:04.117Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"66dff087-454f-4a59-937b-97a99e605a41","DeletionTimestamp":"2026-11-03 15:41:03 +0000 UTC"}
{"level":"info","ts":"2026-11-03T15:41:04.117Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"66dff087-454f-4a59-937b-97a99e605a41","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-11-03T15:41:04.179Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"73a9bc15-63e0-46d5-96b4-0d14d5805b4e","DeletionTimestamp":"2026-11-03 15:41:03 +0000 UTC"}
{"level":"error","ts":"2026-11-03T15:41:04.179Z","logger":"controllers.RayJob","msg":"Failed to get RayCluster","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"73a9bc15-63e0-46d5-96b4-0d14d5805b4e","error":"RayCluster.ray.io \"ttl-initializing-vl6x2\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:120\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-03T15:41:04.179Z","logger":"controllers.RayJob","msg":"Failed to stop job for RayJob","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"73a9bc15-63e0-46d5-96b4-0d14d5805b4e","error":"Post \"http:///api/jobs/ttl-initializing-jj9hd/stop\": http: no Host in request URL","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:128\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:41:04.179Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"73a9bc15-63e0-46d5-96b4-0d14d5805b4e","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-11-03T15:41:04.240Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"2f9da8ee-6c6c-40a5-be4e-aa27e091f631","DeletionTimestamp":"2026-11-03 15:41:03 +0000 UTC"}
{"level":"error","ts":"2026-11-03T15:41:04.240Z","logger":"controllers.RayJob","msg":"Failed to get RayCluster","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"2f9da8ee-6c6c-40a5-be4e-aa27e091f631","error":"RayCluster.ray.io \"ttl-waiting-kjf64\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:120\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-03T15:41:04.243Z","logger":"controllers.RayJob","msg":"Failed to stop job for RayJob","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"2f9da8ee-6c6c-40a5-be4e-aa27e091f631","error":"Post \"http://ttl-waiting-kjf64-head-svc.test-ns-kbx4w.svc.cluster.local:8265/api/jobs//stop\": dial tcp: lookup ttl-waiting-kjf64-head-svc.test-ns-kbx4w.svc.cluster.local on 172.31.0.10:53: no such host","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:128\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:41:04.243Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"2f9da8ee-6c6c-40a5-be4e-aa27e091f631","finalizer":"ray.io/rayjob-finalizer"}
{"level":"info","ts":"2026-11-03T15:41:04.306Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"counter-auth","namespace":"test-ns-kbx4w"},"reconcileID":"c57231db-a6a8-4f6e-b3e3-04458226a69c"}
{"level":"info","ts":"2026-11-03T15:41:04.306Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"long-running","namespace":"test-ns-kbx4w"},"reconcileID":"25d68d56-c292-4be8-8f08-71e5df0f9351"}
{"level":"info","ts":"2026-11-03T15:41:04.306Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"submitter-timeout","namespace":"test-ns-kbx4w"},"reconcileID":"dc83cb43-c503-4f77-91ab-b43f17e11921"}
{"level":"info","ts":"2026-11-03T15:41:04.306Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"ttl-initializing","namespace":"test-ns-kbx4w"},"reconcileID":"60bffb00-293d-4c21-b4d5-fa8aba6ec72c"}
{"level":"info","ts":"2026-11-03T15:41:04.306Z","logger":"controllers.RayJob","msg":"RayJob resource not found.","RayJob":{"name":"ttl-waiting","namespace":"test-ns-kbx4w"},"reconcileID":"695cf1f6-f5b1-4089-90a5-ec8471140ad5"}
{"level":"info","ts":"2026-11-03T15:41:32.411Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"2077fa92-62b5-4426-8d77-23a487268d31","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:42:28.068Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"de89b3ea-1782-46b6-886c-5a22f5af7503"}
{"level":"info","ts":"2026-11-03T15:42:28.068Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"de89b3ea-1782-46b6-886c-5a22f5af7503"}
{"level":"info","ts":"2026-11-03T15:42:28.068Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"de89b3ea-1782-46b6-886c-5a22f5af7503","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:42:28.068Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"de89b3ea-1782-46b6-886c-5a22f5af7503","Found 1 head Pod":"ray-cluster-head-9lzd6","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:42:28.068Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"de89b3ea-1782-46b6-886c-5a22f5af7503","head Pod":"ray-cluster-head-9lzd6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-9lzd6. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:42:28.068Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"de89b3ea-1782-46b6-886c-5a22f5af7503","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-11-03T15:42:28.068Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"de89b3ea-1782-46b6-886c-5a22f5af7503","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:42:28.068Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"de89b3ea-1782-46b6-886c-5a22f5af7503","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-11-03T15:42:28.068Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"de89b3ea-1782-46b6-886c-5a22f5af7503","group":"small-group","indices":{}}
{"level":"info","ts":"2026-11-03T15:42:28.068Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"de89b3ea-1782-46b6-886c-5a22f5af7503","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:42:28.068Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"de89b3ea-1782-46b6-886c-5a22f5af7503","podName":"ray-cluster-head-9lzd6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:42:28.068Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"de89b3ea-1782-46b6-886c-5a22f5af7503","podName":"ray-cluster-head-9lzd6","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:42:28.068Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"de89b3ea-1782-46b6-886c-5a22f5af7503","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:42:28.068Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"de89b3ea-1782-46b6-886c-5a22f5af7503","seconds":300}
{"level":"info","ts":"2026-11-03T15:42:29.022Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"66ad76b6-6241-4fb4-8a2a-b22b24e92857"}
{"level":"info","ts":"2026-11-03T15:42:29.022Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"66ad76b6-6241-4fb4-8a2a-b22b24e92857"}
{"level":"info","ts":"2026-11-03T15:42:29.022Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"66ad76b6-6241-4fb4-8a2a-b22b24e92857","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:42:29.022Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"66ad76b6-6241-4fb4-8a2a-b22b24e92857","Found 1 head Pod":"ray-cluster-head-9lzd6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:42:29.023Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"66ad76b6-6241-4fb4-8a2a-b22b24e92857","head Pod":"ray-cluster-head-9lzd6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-9lzd6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:42:29.023Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"66ad76b6-6241-4fb4-8a2a-b22b24e92857","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-11-03T15:42:29.023Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"66ad76b6-6241-4fb4-8a2a-b22b24e92857","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:42:29.023Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"66ad76b6-6241-4fb4-8a2a-b22b24e92857","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-11-03T15:42:29.023Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"66ad76b6-6241-4fb4-8a2a-b22b24e92857","group":"small-group","indices":{}}
{"level":"info","ts":"2026-11-03T15:42:29.023Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"66ad76b6-6241-4fb4-8a2a-b22b24e92857","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:42:29.023Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"66ad76b6-6241-4fb4-8a2a-b22b24e92857","podName":"ray-cluster-head-9lzd6","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:40:59Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:42:29.023Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"66ad76b6-6241-4fb4-8a2a-b22b24e92857","podName":"ray-cluster-head-9lzd6","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:40:59Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:42:29.023Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"66ad76b6-6241-4fb4-8a2a-b22b24e92857","name":"ray-cluster","old status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:41:01Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.56","serviceIP":"10.134.0.56","podName":"ray-cluster-head-9lzd6","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:40:56Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"PodInitializing","message":"containers with unready status: [ray-head autoscaler kube-rbac-proxy]; autoscaler: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:42:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.56","serviceIP":"10.134.0.56","podName":"ray-cluster-head-9lzd6","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:40:56Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:42:29.083Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"03b22072-7991-400e-90ee-29bd1c23a9c7","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:42:31.084Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b93343bd-e086-4530-b8d6-a729c1ba05e5"}
{"level":"info","ts":"2026-11-03T15:42:31.084Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b93343bd-e086-4530-b8d6-a729c1ba05e5"}
{"level":"info","ts":"2026-11-03T15:42:31.084Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b93343bd-e086-4530-b8d6-a729c1ba05e5","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:42:31.084Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b93343bd-e086-4530-b8d6-a729c1ba05e5","Found 1 head Pod":"ray-cluster-head-9lzd6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:42:31.084Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b93343bd-e086-4530-b8d6-a729c1ba05e5","head Pod":"ray-cluster-head-9lzd6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-9lzd6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:42:31.084Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b93343bd-e086-4530-b8d6-a729c1ba05e5","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-11-03T15:42:31.084Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b93343bd-e086-4530-b8d6-a729c1ba05e5","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:42:31.084Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b93343bd-e086-4530-b8d6-a729c1ba05e5","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-11-03T15:42:31.084Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b93343bd-e086-4530-b8d6-a729c1ba05e5","group":"small-group","indices":{}}
{"level":"info","ts":"2026-11-03T15:42:31.084Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b93343bd-e086-4530-b8d6-a729c1ba05e5","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:42:31.084Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b93343bd-e086-4530-b8d6-a729c1ba05e5","podName":"ray-cluster-head-9lzd6","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:40:59Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:42:31.084Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b93343bd-e086-4530-b8d6-a729c1ba05e5","podName":"ray-cluster-head-9lzd6","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:40:59Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:42:31.084Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b93343bd-e086-4530-b8d6-a729c1ba05e5","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:42:31.084Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b93343bd-e086-4530-b8d6-a729c1ba05e5","seconds":300}
{"level":"info","ts":"2026-11-03T15:42:45.080Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"ff8b353a-ce1f-4588-bd3e-300ec7693123"}
{"level":"info","ts":"2026-11-03T15:42:45.080Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"ff8b353a-ce1f-4588-bd3e-300ec7693123"}
{"level":"info","ts":"2026-11-03T15:42:45.080Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"ff8b353a-ce1f-4588-bd3e-300ec7693123","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:42:45.080Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"ff8b353a-ce1f-4588-bd3e-300ec7693123","Found 1 head Pod":"ray-cluster-head-9lzd6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:42:45.080Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"ff8b353a-ce1f-4588-bd3e-300ec7693123","head Pod":"ray-cluster-head-9lzd6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-9lzd6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:42:45.080Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"ff8b353a-ce1f-4588-bd3e-300ec7693123","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-11-03T15:42:45.080Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"ff8b353a-ce1f-4588-bd3e-300ec7693123","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:42:45.080Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"ff8b353a-ce1f-4588-bd3e-300ec7693123","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-11-03T15:42:45.080Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"ff8b353a-ce1f-4588-bd3e-300ec7693123","group":"small-group","indices":{}}
{"level":"info","ts":"2026-11-03T15:42:45.080Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"ff8b353a-ce1f-4588-bd3e-300ec7693123","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:42:45.081Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"ff8b353a-ce1f-4588-bd3e-300ec7693123","name":"ray-cluster","old status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:42:29Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.56","serviceIP":"10.134.0.56","podName":"ray-cluster-head-9lzd6","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:40:56Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:42:45Z","stateTransitionTimes":{"ready":"2026-11-03T15:42:45Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.56","serviceIP":"10.134.0.56","podName":"ray-cluster-head-9lzd6","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:40:56Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:42:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:42:45Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:42:45.140Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b7cb374b-49f7-43cc-858b-655c2d7992d9","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:42:47.140Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"5395a107-2c52-4625-8fab-0714ce46d688"}
{"level":"info","ts":"2026-11-03T15:42:47.140Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"5395a107-2c52-4625-8fab-0714ce46d688"}
{"level":"info","ts":"2026-11-03T15:42:47.140Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"5395a107-2c52-4625-8fab-0714ce46d688","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:42:47.141Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"5395a107-2c52-4625-8fab-0714ce46d688","Found 1 head Pod":"ray-cluster-head-9lzd6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:42:47.141Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"5395a107-2c52-4625-8fab-0714ce46d688","head Pod":"ray-cluster-head-9lzd6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-9lzd6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:42:47.141Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"5395a107-2c52-4625-8fab-0714ce46d688","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-11-03T15:42:47.141Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"5395a107-2c52-4625-8fab-0714ce46d688","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:42:47.141Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"5395a107-2c52-4625-8fab-0714ce46d688","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-11-03T15:42:47.141Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"5395a107-2c52-4625-8fab-0714ce46d688","group":"small-group","indices":{}}
{"level":"info","ts":"2026-11-03T15:42:47.141Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"5395a107-2c52-4625-8fab-0714ce46d688","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:42:47.141Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"5395a107-2c52-4625-8fab-0714ce46d688","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:42:47.141Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"5395a107-2c52-4625-8fab-0714ce46d688","seconds":300}
{"level":"info","ts":"2026-11-03T15:42:53.425Z","logger":"raycluster-resource","msg":"default","name":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:42:53.425Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"ray-cluster","namespace":"test-ns-4jsqd"}
{"level":"info","ts":"2026-11-03T15:42:53.602Z","logger":"raycluster-resource","msg":"validate update","name":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:42:53.657Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"7a398448-54ab-47bd-bef9-396cb3efd25a","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:42:53.657Z","logger":"controllers.NetworkPolicy","msg":"Reconciling NetworkPolicies for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"6692ec7d-2127-4c3a-8ec9-67db5b24a6a7","cluster":"ray-cluster","namespace":"test-ns-4jsqd"}
{"level":"info","ts":"2026-11-03T15:42:53.658Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"c663dea4-0b02-4b35-a1a7-b10d6449e5e2"}
{"level":"info","ts":"2026-11-03T15:42:53.658Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"c663dea4-0b02-4b35-a1a7-b10d6449e5e2"}
{"level":"info","ts":"2026-11-03T15:42:53.658Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"namespace":"test-ns-4jsqd","name":"ray-cluster","reconcileID":"5a130d8f-5975-49ac-9373-06209c2d1e75","namespacedName":{"name":"ray-cluster","namespace":"test-ns-4jsqd"}}
{"level":"info","ts":"2026-11-03T15:42:53.658Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"c663dea4-0b02-4b35-a1a7-b10d6449e5e2","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:42:53.658Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"namespace":"test-ns-4jsqd","name":"ray-cluster","reconcileID":"5a130d8f-5975-49ac-9373-06209c2d1e75","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:42:53.658Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"namespace":"test-ns-4jsqd","name":"ray-cluster","reconcileID":"5a130d8f-5975-49ac-9373-06209c2d1e75","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:42:53.658Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"c663dea4-0b02-4b35-a1a7-b10d6449e5e2","Found 1 head Pod":"ray-cluster-head-9lzd6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:42:53.658Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"c663dea4-0b02-4b35-a1a7-b10d6449e5e2","head Pod":"ray-cluster-head-9lzd6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-9lzd6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:42:53.658Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"c663dea4-0b02-4b35-a1a7-b10d6449e5e2","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":1}
{"level":"info","ts":"2026-11-03T15:42:53.658Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"c663dea4-0b02-4b35-a1a7-b10d6449e5e2","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:42:53.658Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"c663dea4-0b02-4b35-a1a7-b10d6449e5e2","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-11-03T15:42:53.658Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"c663dea4-0b02-4b35-a1a7-b10d6449e5e2","group":"small-group","indices":{}}
{"level":"info","ts":"2026-11-03T15:42:53.658Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"c663dea4-0b02-4b35-a1a7-b10d6449e5e2","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-11-03T15:42:53.760Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"c663dea4-0b02-4b35-a1a7-b10d6449e5e2","creating worker for group":"small-group","index":0,"total":1,"replicaIndex":0}
{"level":"info","ts":"2026-11-03T15:42:53.760Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"c663dea4-0b02-4b35-a1a7-b10d6449e5e2","nodeType":"worker","rayStartParams":{"address":"ray-cluster-head-svc.test-ns-4jsqd.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"1G"},"requests":{"cpu":"1","memory":"1G"}}}
{"level":"info","ts":"2026-11-03T15:42:53.760Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"c663dea4-0b02-4b35-a1a7-b10d6449e5e2","rayStartCmd":"ray start  --address=ray-cluster-head-svc.test-ns-4jsqd.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=1000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-11-03T15:42:53.760Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"c663dea4-0b02-4b35-a1a7-b10d6449e5e2","rayNodeType":"worker","generatedCmd":"ulimit -n ${RAY_START_ULIMIT_OPEN_FILES:-65536}; ray start  --address=ray-cluster-head-svc.test-ns-4jsqd.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=1000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-11-03T15:42:53.760Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"c663dea4-0b02-4b35-a1a7-b10d6449e5e2","enabled":true}
{"level":"info","ts":"2026-11-03T15:42:53.779Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"namespace":"test-ns-4jsqd","name":"ray-cluster","reconcileID":"5a130d8f-5975-49ac-9373-06209c2d1e75","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:42:53.807Z","logger":"controllers.NetworkPolicy","msg":"Successfully reconciled NetworkPolicies for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"6692ec7d-2127-4c3a-8ec9-67db5b24a6a7","cluster":"ray-cluster","namespace":"test-ns-4jsqd"}
{"level":"info","ts":"2026-11-03T15:42:53.830Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"c663dea4-0b02-4b35-a1a7-b10d6449e5e2","name":"ray-cluster-small-group-worker-vwxmp"}
{"level":"info","ts":"2026-11-03T15:42:53.831Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"c663dea4-0b02-4b35-a1a7-b10d6449e5e2","podName":"ray-cluster-small-group-worker-vwxmp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:42:53.861Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"c663dea4-0b02-4b35-a1a7-b10d6449e5e2","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:42:45Z","stateTransitionTimes":{"ready":"2026-11-03T15:42:45Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.56","serviceIP":"10.134.0.56","podName":"ray-cluster-head-9lzd6","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:40:56Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:42:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:42:45Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:42:53Z","stateTransitionTimes":{"ready":"2026-11-03T15:42:45Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.56","serviceIP":"10.134.0.56","podName":"ray-cluster-head-9lzd6","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:40:56Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:42:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:42:45Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":2}}
{"level":"info","ts":"2026-11-03T15:42:53.916Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"c663dea4-0b02-4b35-a1a7-b10d6449e5e2","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-4jsqd","name":"ray-cluster"}}
{"level":"info","ts":"2026-11-03T15:42:53.916Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"c663dea4-0b02-4b35-a1a7-b10d6449e5e2"}
{"level":"error","ts":"2026-11-03T15:42:53.916Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"c663dea4-0b02-4b35-a1a7-b10d6449e5e2","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:42:53.917Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"03f994c6-1ef5-4a2c-a156-8ceb5f61c124"}
{"level":"info","ts":"2026-11-03T15:42:53.917Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"03f994c6-1ef5-4a2c-a156-8ceb5f61c124"}
{"level":"info","ts":"2026-11-03T15:42:53.917Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"03f994c6-1ef5-4a2c-a156-8ceb5f61c124","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:42:53.917Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"03f994c6-1ef5-4a2c-a156-8ceb5f61c124","Found 1 head Pod":"ray-cluster-head-9lzd6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:42:53.917Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"03f994c6-1ef5-4a2c-a156-8ceb5f61c124","head Pod":"ray-cluster-head-9lzd6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-9lzd6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:42:53.917Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"03f994c6-1ef5-4a2c-a156-8ceb5f61c124","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":1}
{"level":"info","ts":"2026-11-03T15:42:53.917Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"03f994c6-1ef5-4a2c-a156-8ceb5f61c124","worker Pod":"ray-cluster-small-group-worker-vwxmp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-vwxmp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:42:53.917Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"03f994c6-1ef5-4a2c-a156-8ceb5f61c124","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:42:53.917Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"03f994c6-1ef5-4a2c-a156-8ceb5f61c124","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:42:53.917Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"03f994c6-1ef5-4a2c-a156-8ceb5f61c124","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:42:53.917Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"03f994c6-1ef5-4a2c-a156-8ceb5f61c124","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:42:53.917Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"03f994c6-1ef5-4a2c-a156-8ceb5f61c124","podName":"ray-cluster-small-group-worker-vwxmp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:42:53.917Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"03f994c6-1ef5-4a2c-a156-8ceb5f61c124","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:42:45Z","stateTransitionTimes":{"ready":"2026-11-03T15:42:45Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.56","serviceIP":"10.134.0.56","podName":"ray-cluster-head-9lzd6","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":2,"lastTransitionTime":"2026-11-03T15:40:56Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:42:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:42:45Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:42:53Z","stateTransitionTimes":{"ready":"2026-11-03T15:42:45Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.56","serviceIP":"10.134.0.56","podName":"ray-cluster-head-9lzd6","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":2,"lastTransitionTime":"2026-11-03T15:40:56Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:42:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:42:45Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":2}}
{"level":"info","ts":"2026-11-03T15:42:54.021Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"a1f09af4-7269-424c-b189-297b788767d3"}
{"level":"info","ts":"2026-11-03T15:42:54.021Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"a1f09af4-7269-424c-b189-297b788767d3"}
{"level":"info","ts":"2026-11-03T15:42:54.021Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"a1f09af4-7269-424c-b189-297b788767d3","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:42:54.021Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"a1f09af4-7269-424c-b189-297b788767d3","Found 1 head Pod":"ray-cluster-head-9lzd6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:42:54.021Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"a1f09af4-7269-424c-b189-297b788767d3","head Pod":"ray-cluster-head-9lzd6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-9lzd6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:42:54.021Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"a1f09af4-7269-424c-b189-297b788767d3","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":1}
{"level":"info","ts":"2026-11-03T15:42:54.021Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"a1f09af4-7269-424c-b189-297b788767d3","worker Pod":"ray-cluster-small-group-worker-vwxmp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-vwxmp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:42:54.021Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"a1f09af4-7269-424c-b189-297b788767d3","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:42:54.021Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"a1f09af4-7269-424c-b189-297b788767d3","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:42:54.021Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"a1f09af4-7269-424c-b189-297b788767d3","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:42:54.021Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"a1f09af4-7269-424c-b189-297b788767d3","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:42:54.022Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"a1f09af4-7269-424c-b189-297b788767d3","podName":"ray-cluster-small-group-worker-vwxmp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:42:54.022Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"a1f09af4-7269-424c-b189-297b788767d3","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:42:45Z","stateTransitionTimes":{"ready":"2026-11-03T15:42:45Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.56","serviceIP":"10.134.0.56","podName":"ray-cluster-head-9lzd6","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":2,"lastTransitionTime":"2026-11-03T15:40:56Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:42:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:42:45Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:42:54Z","stateTransitionTimes":{"ready":"2026-11-03T15:42:45Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.56","serviceIP":"10.134.0.56","podName":"ray-cluster-head-9lzd6","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":2,"lastTransitionTime":"2026-11-03T15:40:56Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:42:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:42:45Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":2}}
{"level":"info","ts":"2026-11-03T15:42:54.076Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"a1f09af4-7269-424c-b189-297b788767d3","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-4jsqd","name":"ray-cluster"}}
{"level":"info","ts":"2026-11-03T15:42:54.076Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"a1f09af4-7269-424c-b189-297b788767d3"}
{"level":"error","ts":"2026-11-03T15:42:54.076Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"a1f09af4-7269-424c-b189-297b788767d3","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:42:54.081Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"cf702ec5-3627-460d-8589-f82dbb06b7f3"}
{"level":"info","ts":"2026-11-03T15:42:54.081Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"cf702ec5-3627-460d-8589-f82dbb06b7f3"}
{"level":"info","ts":"2026-11-03T15:42:54.081Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"cf702ec5-3627-460d-8589-f82dbb06b7f3","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:42:54.082Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"cf702ec5-3627-460d-8589-f82dbb06b7f3","Found 1 head Pod":"ray-cluster-head-9lzd6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:42:54.082Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"cf702ec5-3627-460d-8589-f82dbb06b7f3","head Pod":"ray-cluster-head-9lzd6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-9lzd6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:42:54.082Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"cf702ec5-3627-460d-8589-f82dbb06b7f3","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":1}
{"level":"info","ts":"2026-11-03T15:42:54.082Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"cf702ec5-3627-460d-8589-f82dbb06b7f3","worker Pod":"ray-cluster-small-group-worker-vwxmp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-vwxmp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:42:54.082Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"cf702ec5-3627-460d-8589-f82dbb06b7f3","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:42:54.082Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"cf702ec5-3627-460d-8589-f82dbb06b7f3","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:42:54.082Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"cf702ec5-3627-460d-8589-f82dbb06b7f3","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:42:54.082Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"cf702ec5-3627-460d-8589-f82dbb06b7f3","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:42:54.082Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"cf702ec5-3627-460d-8589-f82dbb06b7f3","podName":"ray-cluster-small-group-worker-vwxmp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:42:54.082Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"cf702ec5-3627-460d-8589-f82dbb06b7f3","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:42:54.082Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"cf702ec5-3627-460d-8589-f82dbb06b7f3","seconds":300}
{"level":"info","ts":"2026-11-03T15:42:54.477Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"3012aa07-57c1-4ae0-9263-8c3199450870","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:42:54.887Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b19ee453-4264-4ef4-8f51-b7888bd5b281"}
{"level":"info","ts":"2026-11-03T15:42:54.887Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b19ee453-4264-4ef4-8f51-b7888bd5b281"}
{"level":"info","ts":"2026-11-03T15:42:54.887Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b19ee453-4264-4ef4-8f51-b7888bd5b281","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:42:54.887Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b19ee453-4264-4ef4-8f51-b7888bd5b281","Found 1 head Pod":"ray-cluster-head-9lzd6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:42:54.887Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b19ee453-4264-4ef4-8f51-b7888bd5b281","head Pod":"ray-cluster-head-9lzd6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-9lzd6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:42:54.887Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b19ee453-4264-4ef4-8f51-b7888bd5b281","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":1}
{"level":"info","ts":"2026-11-03T15:42:54.887Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b19ee453-4264-4ef4-8f51-b7888bd5b281","worker Pod":"ray-cluster-small-group-worker-vwxmp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-vwxmp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:42:54.887Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b19ee453-4264-4ef4-8f51-b7888bd5b281","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:42:54.887Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b19ee453-4264-4ef4-8f51-b7888bd5b281","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:42:54.887Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b19ee453-4264-4ef4-8f51-b7888bd5b281","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:42:54.887Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b19ee453-4264-4ef4-8f51-b7888bd5b281","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:42:54.887Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b19ee453-4264-4ef4-8f51-b7888bd5b281","podName":"ray-cluster-small-group-worker-vwxmp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:42:54.887Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b19ee453-4264-4ef4-8f51-b7888bd5b281","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:42:54.887Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b19ee453-4264-4ef4-8f51-b7888bd5b281","seconds":300}
{"level":"info","ts":"2026-11-03T15:42:55.822Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"0e2f7df2-4c62-4a7e-b5aa-f6534f0a2233"}
{"level":"info","ts":"2026-11-03T15:42:55.822Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"0e2f7df2-4c62-4a7e-b5aa-f6534f0a2233"}
{"level":"info","ts":"2026-11-03T15:42:55.822Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"0e2f7df2-4c62-4a7e-b5aa-f6534f0a2233","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:42:55.822Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"0e2f7df2-4c62-4a7e-b5aa-f6534f0a2233","Found 1 head Pod":"ray-cluster-head-9lzd6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:42:55.822Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"0e2f7df2-4c62-4a7e-b5aa-f6534f0a2233","head Pod":"ray-cluster-head-9lzd6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-9lzd6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:42:55.822Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"0e2f7df2-4c62-4a7e-b5aa-f6534f0a2233","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":1}
{"level":"info","ts":"2026-11-03T15:42:55.822Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"0e2f7df2-4c62-4a7e-b5aa-f6534f0a2233","worker Pod":"ray-cluster-small-group-worker-vwxmp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-vwxmp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:42:55.822Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"0e2f7df2-4c62-4a7e-b5aa-f6534f0a2233","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:42:55.822Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"0e2f7df2-4c62-4a7e-b5aa-f6534f0a2233","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:42:55.822Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"0e2f7df2-4c62-4a7e-b5aa-f6534f0a2233","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:42:55.823Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"0e2f7df2-4c62-4a7e-b5aa-f6534f0a2233","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:42:55.823Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"0e2f7df2-4c62-4a7e-b5aa-f6534f0a2233","podName":"ray-cluster-small-group-worker-vwxmp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:42:55.823Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"0e2f7df2-4c62-4a7e-b5aa-f6534f0a2233","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:42:55.823Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"0e2f7df2-4c62-4a7e-b5aa-f6534f0a2233","seconds":300}
{"level":"info","ts":"2026-11-03T15:43:00.339Z","logger":"raycluster-resource","msg":"default","name":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:43:00.339Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"ray-cluster","namespace":"test-ns-4jsqd"}
{"level":"info","ts":"2026-11-03T15:43:00.396Z","logger":"raycluster-resource","msg":"validate update","name":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:43:00.451Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"8a1704ee-3da4-490e-a1f0-077f1654e58d","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:43:00.451Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"namespace":"test-ns-4jsqd","name":"ray-cluster","reconcileID":"8468bbda-ac09-4eb5-a6c4-d56837015fc0","namespacedName":{"name":"ray-cluster","namespace":"test-ns-4jsqd"}}
{"level":"info","ts":"2026-11-03T15:43:00.451Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"namespace":"test-ns-4jsqd","name":"ray-cluster","reconcileID":"8468bbda-ac09-4eb5-a6c4-d56837015fc0","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:43:00.451Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"namespace":"test-ns-4jsqd","name":"ray-cluster","reconcileID":"8468bbda-ac09-4eb5-a6c4-d56837015fc0","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:43:00.451Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"72a1e5e3-a896-4a06-a2f4-36878898809b"}
{"level":"info","ts":"2026-11-03T15:43:00.451Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"72a1e5e3-a896-4a06-a2f4-36878898809b"}
{"level":"info","ts":"2026-11-03T15:43:00.451Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"72a1e5e3-a896-4a06-a2f4-36878898809b","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:43:00.451Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"72a1e5e3-a896-4a06-a2f4-36878898809b","Found 1 head Pod":"ray-cluster-head-9lzd6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:43:00.451Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"72a1e5e3-a896-4a06-a2f4-36878898809b","head Pod":"ray-cluster-head-9lzd6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-9lzd6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:43:00.451Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"72a1e5e3-a896-4a06-a2f4-36878898809b","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-11-03T15:43:00.451Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"72a1e5e3-a896-4a06-a2f4-36878898809b","worker Pod":"ray-cluster-small-group-worker-vwxmp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-vwxmp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:43:00.451Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"72a1e5e3-a896-4a06-a2f4-36878898809b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:43:00.451Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"72a1e5e3-a896-4a06-a2f4-36878898809b","workerReplicas":2,"NumOfHosts":1,"runningPods":1,"diff":1}
{"level":"info","ts":"2026-11-03T15:43:00.451Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"72a1e5e3-a896-4a06-a2f4-36878898809b","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:43:00.451Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"72a1e5e3-a896-4a06-a2f4-36878898809b","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-11-03T15:43:00.452Z","logger":"controllers.NetworkPolicy","msg":"Reconciling NetworkPolicies for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"de0f31ae-7882-49f7-9699-d8ee9f2f7e57","cluster":"ray-cluster","namespace":"test-ns-4jsqd"}
{"level":"info","ts":"2026-11-03T15:43:00.548Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"namespace":"test-ns-4jsqd","name":"ray-cluster","reconcileID":"8468bbda-ac09-4eb5-a6c4-d56837015fc0","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:43:00.551Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"72a1e5e3-a896-4a06-a2f4-36878898809b","creating worker for group":"small-group","index":0,"total":1,"replicaIndex":1}
{"level":"info","ts":"2026-11-03T15:43:00.551Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"72a1e5e3-a896-4a06-a2f4-36878898809b","nodeType":"worker","rayStartParams":{"address":"ray-cluster-head-svc.test-ns-4jsqd.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"1G"},"requests":{"cpu":"1","memory":"1G"}}}
{"level":"info","ts":"2026-11-03T15:43:00.551Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"72a1e5e3-a896-4a06-a2f4-36878898809b","rayStartCmd":"ray start  --address=ray-cluster-head-svc.test-ns-4jsqd.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=1000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-11-03T15:43:00.551Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"72a1e5e3-a896-4a06-a2f4-36878898809b","rayNodeType":"worker","generatedCmd":"ulimit -n ${RAY_START_ULIMIT_OPEN_FILES:-65536}; ray start  --address=ray-cluster-head-svc.test-ns-4jsqd.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=1000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-11-03T15:43:00.551Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"72a1e5e3-a896-4a06-a2f4-36878898809b","enabled":true}
{"level":"info","ts":"2026-11-03T15:43:00.612Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"72a1e5e3-a896-4a06-a2f4-36878898809b","name":"ray-cluster-small-group-worker-hhv2f"}
{"level":"info","ts":"2026-11-03T15:43:00.613Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"72a1e5e3-a896-4a06-a2f4-36878898809b","podName":"ray-cluster-small-group-worker-hhv2f","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:43:00.613Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"72a1e5e3-a896-4a06-a2f4-36878898809b","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:42:53Z","stateTransitionTimes":{"ready":"2026-11-03T15:42:45Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.56","serviceIP":"10.134.0.56","podName":"ray-cluster-head-9lzd6","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":2,"lastTransitionTime":"2026-11-03T15:40:56Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:42:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:42:45Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":2},"new status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:43:00Z","stateTransitionTimes":{"ready":"2026-11-03T15:42:45Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.56","serviceIP":"10.134.0.56","podName":"ray-cluster-head-9lzd6","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":2,"lastTransitionTime":"2026-11-03T15:40:56Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:42:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:42:45Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3}}
{"level":"info","ts":"2026-11-03T15:43:00.614Z","logger":"controllers.NetworkPolicy","msg":"Successfully reconciled NetworkPolicies for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"de0f31ae-7882-49f7-9699-d8ee9f2f7e57","cluster":"ray-cluster","namespace":"test-ns-4jsqd"}
{"level":"info","ts":"2026-11-03T15:43:00.667Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"72a1e5e3-a896-4a06-a2f4-36878898809b","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-4jsqd","name":"ray-cluster"}}
{"level":"info","ts":"2026-11-03T15:43:00.667Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"72a1e5e3-a896-4a06-a2f4-36878898809b"}
{"level":"error","ts":"2026-11-03T15:43:00.667Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"72a1e5e3-a896-4a06-a2f4-36878898809b","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:43:00.668Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"f38c6cc4-f0fe-47cc-9fef-73b3adf6b74e"}
{"level":"info","ts":"2026-11-03T15:43:00.668Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"f38c6cc4-f0fe-47cc-9fef-73b3adf6b74e"}
{"level":"info","ts":"2026-11-03T15:43:00.668Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"f38c6cc4-f0fe-47cc-9fef-73b3adf6b74e","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:43:00.668Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"f38c6cc4-f0fe-47cc-9fef-73b3adf6b74e","Found 1 head Pod":"ray-cluster-head-9lzd6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:43:00.668Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"f38c6cc4-f0fe-47cc-9fef-73b3adf6b74e","head Pod":"ray-cluster-head-9lzd6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-9lzd6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:43:00.668Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"f38c6cc4-f0fe-47cc-9fef-73b3adf6b74e","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-11-03T15:43:00.668Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"f38c6cc4-f0fe-47cc-9fef-73b3adf6b74e","worker Pod":"ray-cluster-small-group-worker-vwxmp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-vwxmp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:43:00.668Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"f38c6cc4-f0fe-47cc-9fef-73b3adf6b74e","worker Pod":"ray-cluster-small-group-worker-hhv2f","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-hhv2f. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:43:00.668Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"f38c6cc4-f0fe-47cc-9fef-73b3adf6b74e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:43:00.668Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"f38c6cc4-f0fe-47cc-9fef-73b3adf6b74e","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-11-03T15:43:00.668Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"f38c6cc4-f0fe-47cc-9fef-73b3adf6b74e","group":"small-group","indices":{"0":true,"1":true}}
{"level":"info","ts":"2026-11-03T15:43:00.668Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"f38c6cc4-f0fe-47cc-9fef-73b3adf6b74e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:43:00.668Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"f38c6cc4-f0fe-47cc-9fef-73b3adf6b74e","podName":"ray-cluster-small-group-worker-vwxmp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:43:00.668Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"f38c6cc4-f0fe-47cc-9fef-73b3adf6b74e","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:42:53Z","stateTransitionTimes":{"ready":"2026-11-03T15:42:45Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.56","serviceIP":"10.134.0.56","podName":"ray-cluster-head-9lzd6","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-11-03T15:40:56Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:42:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:42:45Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":2},"new status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:43:00Z","stateTransitionTimes":{"ready":"2026-11-03T15:42:45Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.56","serviceIP":"10.134.0.56","podName":"ray-cluster-head-9lzd6","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-11-03T15:40:56Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:42:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:42:45Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3}}
{"level":"info","ts":"2026-11-03T15:43:00.729Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"1e509abd-de0a-4c98-a98f-7092f945ac4d"}
{"level":"info","ts":"2026-11-03T15:43:00.729Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"1e509abd-de0a-4c98-a98f-7092f945ac4d"}
{"level":"info","ts":"2026-11-03T15:43:00.729Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"1e509abd-de0a-4c98-a98f-7092f945ac4d","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:43:00.729Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"1e509abd-de0a-4c98-a98f-7092f945ac4d","Found 1 head Pod":"ray-cluster-head-9lzd6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:43:00.729Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"1e509abd-de0a-4c98-a98f-7092f945ac4d","head Pod":"ray-cluster-head-9lzd6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-9lzd6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:43:00.729Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"1e509abd-de0a-4c98-a98f-7092f945ac4d","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-11-03T15:43:00.729Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"1e509abd-de0a-4c98-a98f-7092f945ac4d","worker Pod":"ray-cluster-small-group-worker-vwxmp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-vwxmp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:43:00.729Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"1e509abd-de0a-4c98-a98f-7092f945ac4d","worker Pod":"ray-cluster-small-group-worker-hhv2f","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-hhv2f. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:43:00.729Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"1e509abd-de0a-4c98-a98f-7092f945ac4d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:43:00.729Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"1e509abd-de0a-4c98-a98f-7092f945ac4d","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-11-03T15:43:00.729Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"1e509abd-de0a-4c98-a98f-7092f945ac4d","group":"small-group","indices":{"0":true,"1":true}}
{"level":"info","ts":"2026-11-03T15:43:00.729Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"1e509abd-de0a-4c98-a98f-7092f945ac4d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:43:00.729Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"1e509abd-de0a-4c98-a98f-7092f945ac4d","podName":"ray-cluster-small-group-worker-hhv2f","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:43:00.729Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"1e509abd-de0a-4c98-a98f-7092f945ac4d","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:42:53Z","stateTransitionTimes":{"ready":"2026-11-03T15:42:45Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.56","serviceIP":"10.134.0.56","podName":"ray-cluster-head-9lzd6","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-11-03T15:40:56Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:42:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:42:45Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":2},"new status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:43:00Z","stateTransitionTimes":{"ready":"2026-11-03T15:42:45Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.56","serviceIP":"10.134.0.56","podName":"ray-cluster-head-9lzd6","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-11-03T15:40:56Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:42:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:42:45Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3}}
{"level":"info","ts":"2026-11-03T15:43:00.784Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"1e509abd-de0a-4c98-a98f-7092f945ac4d","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-4jsqd","name":"ray-cluster"}}
{"level":"info","ts":"2026-11-03T15:43:00.784Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"1e509abd-de0a-4c98-a98f-7092f945ac4d"}
{"level":"error","ts":"2026-11-03T15:43:00.784Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"1e509abd-de0a-4c98-a98f-7092f945ac4d","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:43:00.784Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"20de110d-0b59-4a3f-b4b7-fcd2df7364e6"}
{"level":"info","ts":"2026-11-03T15:43:00.784Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"20de110d-0b59-4a3f-b4b7-fcd2df7364e6"}
{"level":"info","ts":"2026-11-03T15:43:00.784Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"20de110d-0b59-4a3f-b4b7-fcd2df7364e6","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:43:00.784Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"20de110d-0b59-4a3f-b4b7-fcd2df7364e6","Found 1 head Pod":"ray-cluster-head-9lzd6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:43:00.784Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"20de110d-0b59-4a3f-b4b7-fcd2df7364e6","head Pod":"ray-cluster-head-9lzd6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-9lzd6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:43:00.784Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"20de110d-0b59-4a3f-b4b7-fcd2df7364e6","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-11-03T15:43:00.784Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"20de110d-0b59-4a3f-b4b7-fcd2df7364e6","worker Pod":"ray-cluster-small-group-worker-vwxmp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-vwxmp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:43:00.784Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"20de110d-0b59-4a3f-b4b7-fcd2df7364e6","worker Pod":"ray-cluster-small-group-worker-hhv2f","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-hhv2f. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:43:00.784Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"20de110d-0b59-4a3f-b4b7-fcd2df7364e6","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:43:00.784Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"20de110d-0b59-4a3f-b4b7-fcd2df7364e6","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-11-03T15:43:00.784Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"20de110d-0b59-4a3f-b4b7-fcd2df7364e6","group":"small-group","indices":{"0":true,"1":true}}
{"level":"info","ts":"2026-11-03T15:43:00.785Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"20de110d-0b59-4a3f-b4b7-fcd2df7364e6","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:43:00.785Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"20de110d-0b59-4a3f-b4b7-fcd2df7364e6","podName":"ray-cluster-small-group-worker-vwxmp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:43:00.785Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"20de110d-0b59-4a3f-b4b7-fcd2df7364e6","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:43:00.785Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"20de110d-0b59-4a3f-b4b7-fcd2df7364e6","seconds":300}
{"level":"info","ts":"2026-11-03T15:43:00.904Z","logger":"controllers.RayClusterMTLS","msg":"Updating worker certificate","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"8a1704ee-3da4-490e-a1f0-077f1654e58d","certificate":"ray-worker-cert-ray-cluster","ipAddresses":["10.132.0.52","127.0.0.1"]}
{"level":"info","ts":"2026-11-03T15:43:01.334Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"8a1704ee-3da4-490e-a1f0-077f1654e58d","certificate":"ray-worker-cert-ray-cluster"}
{"level":"info","ts":"2026-11-03T15:43:01.334Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"8a1704ee-3da4-490e-a1f0-077f1654e58d"}
{"level":"info","ts":"2026-11-03T15:43:01.334Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"3291b0aa-3f16-4fe0-8bc4-0cf854993a05","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:43:01.368Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b23d9854-a017-46db-a8e6-b28868829082"}
{"level":"info","ts":"2026-11-03T15:43:01.368Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b23d9854-a017-46db-a8e6-b28868829082"}
{"level":"info","ts":"2026-11-03T15:43:01.368Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b23d9854-a017-46db-a8e6-b28868829082","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:43:01.368Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b23d9854-a017-46db-a8e6-b28868829082","Found 1 head Pod":"ray-cluster-head-9lzd6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:43:01.368Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b23d9854-a017-46db-a8e6-b28868829082","head Pod":"ray-cluster-head-9lzd6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-9lzd6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:43:01.368Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b23d9854-a017-46db-a8e6-b28868829082","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-11-03T15:43:01.368Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b23d9854-a017-46db-a8e6-b28868829082","worker Pod":"ray-cluster-small-group-worker-hhv2f","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-hhv2f. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:43:01.368Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b23d9854-a017-46db-a8e6-b28868829082","worker Pod":"ray-cluster-small-group-worker-vwxmp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-vwxmp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:43:01.368Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b23d9854-a017-46db-a8e6-b28868829082","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:43:01.368Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b23d9854-a017-46db-a8e6-b28868829082","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-11-03T15:43:01.368Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b23d9854-a017-46db-a8e6-b28868829082","group":"small-group","indices":{"0":true,"1":true}}
{"level":"info","ts":"2026-11-03T15:43:01.368Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b23d9854-a017-46db-a8e6-b28868829082","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:43:01.369Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b23d9854-a017-46db-a8e6-b28868829082","podName":"ray-cluster-small-group-worker-vwxmp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:43:01.369Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b23d9854-a017-46db-a8e6-b28868829082","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:43:01.369Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b23d9854-a017-46db-a8e6-b28868829082","seconds":300}
{"level":"info","ts":"2026-11-03T15:43:01.794Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"dd416add-fa31-49b9-b3a1-c74034fde046"}
{"level":"info","ts":"2026-11-03T15:43:01.794Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"dd416add-fa31-49b9-b3a1-c74034fde046"}
{"level":"info","ts":"2026-11-03T15:43:01.794Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"dd416add-fa31-49b9-b3a1-c74034fde046","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:43:01.794Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"dd416add-fa31-49b9-b3a1-c74034fde046","Found 1 head Pod":"ray-cluster-head-9lzd6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:43:01.795Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"dd416add-fa31-49b9-b3a1-c74034fde046","head Pod":"ray-cluster-head-9lzd6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-9lzd6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:43:01.795Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"dd416add-fa31-49b9-b3a1-c74034fde046","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-11-03T15:43:01.795Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"dd416add-fa31-49b9-b3a1-c74034fde046","worker Pod":"ray-cluster-small-group-worker-vwxmp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-vwxmp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:43:01.795Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"dd416add-fa31-49b9-b3a1-c74034fde046","worker Pod":"ray-cluster-small-group-worker-hhv2f","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-hhv2f. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:43:01.795Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"dd416add-fa31-49b9-b3a1-c74034fde046","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:43:01.795Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"dd416add-fa31-49b9-b3a1-c74034fde046","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-11-03T15:43:01.795Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"dd416add-fa31-49b9-b3a1-c74034fde046","group":"small-group","indices":{"0":true,"1":true}}
{"level":"info","ts":"2026-11-03T15:43:01.795Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"dd416add-fa31-49b9-b3a1-c74034fde046","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:43:01.795Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"dd416add-fa31-49b9-b3a1-c74034fde046","podName":"ray-cluster-small-group-worker-vwxmp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:43:01.795Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"dd416add-fa31-49b9-b3a1-c74034fde046","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:43:01.795Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"dd416add-fa31-49b9-b3a1-c74034fde046","seconds":300}
{"level":"info","ts":"2026-11-03T15:43:01.990Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"3291b0aa-3f16-4fe0-8bc4-0cf854993a05","certificate":"ray-worker-cert-ray-cluster"}
{"level":"info","ts":"2026-11-03T15:43:01.990Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"3291b0aa-3f16-4fe0-8bc4-0cf854993a05"}
{"level":"info","ts":"2026-11-03T15:43:31.991Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"68d93afb-1563-4aee-8bf5-8c95de413265","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:43:32.456Z","logger":"controllers.RayClusterMTLS","msg":"Updating worker certificate","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"68d93afb-1563-4aee-8bf5-8c95de413265","certificate":"ray-worker-cert-ray-cluster","ipAddresses":["10.132.0.52","10.132.0.53","127.0.0.1"]}
{"level":"info","ts":"2026-11-03T15:43:32.766Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"68d93afb-1563-4aee-8bf5-8c95de413265","certificate":"ray-worker-cert-ray-cluster"}
{"level":"info","ts":"2026-11-03T15:43:32.766Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"68d93afb-1563-4aee-8bf5-8c95de413265"}
{"level":"info","ts":"2026-11-03T15:44:02.767Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"a0a4c603-0759-410e-afea-fb253b42281b","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:44:11.126Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"7d7e5c1b-80e0-47a1-9750-e083dc40294b"}
{"level":"info","ts":"2026-11-03T15:44:11.126Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"7d7e5c1b-80e0-47a1-9750-e083dc40294b"}
{"level":"info","ts":"2026-11-03T15:44:11.126Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"7d7e5c1b-80e0-47a1-9750-e083dc40294b","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:44:11.126Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"7d7e5c1b-80e0-47a1-9750-e083dc40294b","Found 1 head Pod":"ray-cluster-head-9lzd6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:44:11.126Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"7d7e5c1b-80e0-47a1-9750-e083dc40294b","head Pod":"ray-cluster-head-9lzd6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-9lzd6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:44:11.126Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"7d7e5c1b-80e0-47a1-9750-e083dc40294b","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-11-03T15:44:11.126Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"7d7e5c1b-80e0-47a1-9750-e083dc40294b","worker Pod":"ray-cluster-small-group-worker-vwxmp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-vwxmp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:44:11.126Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"7d7e5c1b-80e0-47a1-9750-e083dc40294b","worker Pod":"ray-cluster-small-group-worker-hhv2f","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-hhv2f. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:44:11.126Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"7d7e5c1b-80e0-47a1-9750-e083dc40294b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:44:11.126Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"7d7e5c1b-80e0-47a1-9750-e083dc40294b","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-11-03T15:44:11.126Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"7d7e5c1b-80e0-47a1-9750-e083dc40294b","group":"small-group","indices":{"0":true,"1":true}}
{"level":"info","ts":"2026-11-03T15:44:11.126Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"7d7e5c1b-80e0-47a1-9750-e083dc40294b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:44:11.126Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"7d7e5c1b-80e0-47a1-9750-e083dc40294b","podName":"ray-cluster-small-group-worker-vwxmp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:44:11.126Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"7d7e5c1b-80e0-47a1-9750-e083dc40294b","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:44:11.126Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"7d7e5c1b-80e0-47a1-9750-e083dc40294b","seconds":300}
{"level":"info","ts":"2026-11-03T15:44:12.132Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"7741c91c-fff1-44cd-b4e8-7b218d4362d8"}
{"level":"info","ts":"2026-11-03T15:44:12.132Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"7741c91c-fff1-44cd-b4e8-7b218d4362d8"}
{"level":"info","ts":"2026-11-03T15:44:12.132Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"7741c91c-fff1-44cd-b4e8-7b218d4362d8","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:44:12.132Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"7741c91c-fff1-44cd-b4e8-7b218d4362d8","Found 1 head Pod":"ray-cluster-head-9lzd6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:44:12.132Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"7741c91c-fff1-44cd-b4e8-7b218d4362d8","head Pod":"ray-cluster-head-9lzd6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-9lzd6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:44:12.132Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"7741c91c-fff1-44cd-b4e8-7b218d4362d8","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-11-03T15:44:12.132Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"7741c91c-fff1-44cd-b4e8-7b218d4362d8","worker Pod":"ray-cluster-small-group-worker-vwxmp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-vwxmp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:44:12.132Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"7741c91c-fff1-44cd-b4e8-7b218d4362d8","worker Pod":"ray-cluster-small-group-worker-hhv2f","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-hhv2f. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:44:12.132Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"7741c91c-fff1-44cd-b4e8-7b218d4362d8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:44:12.132Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"7741c91c-fff1-44cd-b4e8-7b218d4362d8","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-11-03T15:44:12.132Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"7741c91c-fff1-44cd-b4e8-7b218d4362d8","group":"small-group","indices":{"0":true,"1":true}}
{"level":"info","ts":"2026-11-03T15:44:12.132Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"7741c91c-fff1-44cd-b4e8-7b218d4362d8","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:44:12.133Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"7741c91c-fff1-44cd-b4e8-7b218d4362d8","podName":"ray-cluster-small-group-worker-vwxmp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:44:12.133Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"7741c91c-fff1-44cd-b4e8-7b218d4362d8","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:44:12.133Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"7741c91c-fff1-44cd-b4e8-7b218d4362d8","seconds":300}
{"level":"info","ts":"2026-11-03T15:44:17.151Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"8916a3d8-467d-45d1-ae2e-08e31e9068a5"}
{"level":"info","ts":"2026-11-03T15:44:17.151Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"8916a3d8-467d-45d1-ae2e-08e31e9068a5"}
{"level":"info","ts":"2026-11-03T15:44:17.151Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"8916a3d8-467d-45d1-ae2e-08e31e9068a5","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:44:17.151Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"8916a3d8-467d-45d1-ae2e-08e31e9068a5","Found 1 head Pod":"ray-cluster-head-9lzd6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:44:17.151Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"8916a3d8-467d-45d1-ae2e-08e31e9068a5","head Pod":"ray-cluster-head-9lzd6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-9lzd6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:44:17.151Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"8916a3d8-467d-45d1-ae2e-08e31e9068a5","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-11-03T15:44:17.151Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"8916a3d8-467d-45d1-ae2e-08e31e9068a5","worker Pod":"ray-cluster-small-group-worker-vwxmp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-vwxmp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:44:17.151Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"8916a3d8-467d-45d1-ae2e-08e31e9068a5","worker Pod":"ray-cluster-small-group-worker-hhv2f","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-hhv2f. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:44:17.151Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"8916a3d8-467d-45d1-ae2e-08e31e9068a5","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:44:17.151Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"8916a3d8-467d-45d1-ae2e-08e31e9068a5","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-11-03T15:44:17.151Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"8916a3d8-467d-45d1-ae2e-08e31e9068a5","group":"small-group","indices":{"0":true,"1":true}}
{"level":"info","ts":"2026-11-03T15:44:17.151Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"8916a3d8-467d-45d1-ae2e-08e31e9068a5","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:44:17.151Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"8916a3d8-467d-45d1-ae2e-08e31e9068a5","podName":"ray-cluster-small-group-worker-vwxmp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:44:17.151Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"8916a3d8-467d-45d1-ae2e-08e31e9068a5","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:44:17.151Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"8916a3d8-467d-45d1-ae2e-08e31e9068a5","seconds":300}
{"level":"info","ts":"2026-11-03T15:44:18.163Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"93e09e40-96fc-4742-820f-ea30836eed54"}
{"level":"info","ts":"2026-11-03T15:44:18.163Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"93e09e40-96fc-4742-820f-ea30836eed54"}
{"level":"info","ts":"2026-11-03T15:44:18.163Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"93e09e40-96fc-4742-820f-ea30836eed54","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:44:18.164Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"93e09e40-96fc-4742-820f-ea30836eed54","Found 1 head Pod":"ray-cluster-head-9lzd6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:44:18.164Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"93e09e40-96fc-4742-820f-ea30836eed54","head Pod":"ray-cluster-head-9lzd6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-9lzd6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:44:18.164Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"93e09e40-96fc-4742-820f-ea30836eed54","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-11-03T15:44:18.164Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"93e09e40-96fc-4742-820f-ea30836eed54","worker Pod":"ray-cluster-small-group-worker-vwxmp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-vwxmp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:44:18.164Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"93e09e40-96fc-4742-820f-ea30836eed54","worker Pod":"ray-cluster-small-group-worker-hhv2f","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-hhv2f. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:44:18.164Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"93e09e40-96fc-4742-820f-ea30836eed54","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:44:18.164Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"93e09e40-96fc-4742-820f-ea30836eed54","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-11-03T15:44:18.164Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"93e09e40-96fc-4742-820f-ea30836eed54","group":"small-group","indices":{"0":true,"1":true}}
{"level":"info","ts":"2026-11-03T15:44:18.164Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"93e09e40-96fc-4742-820f-ea30836eed54","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:44:18.164Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"93e09e40-96fc-4742-820f-ea30836eed54","podName":"ray-cluster-small-group-worker-vwxmp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:44:18.164Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"93e09e40-96fc-4742-820f-ea30836eed54","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:44:18.164Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"93e09e40-96fc-4742-820f-ea30836eed54","seconds":300}
{"level":"info","ts":"2026-11-03T15:44:22.176Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"2ef97afd-9c93-48b3-a5a0-49d61f061e64"}
{"level":"info","ts":"2026-11-03T15:44:22.176Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"2ef97afd-9c93-48b3-a5a0-49d61f061e64"}
{"level":"info","ts":"2026-11-03T15:44:22.176Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"2ef97afd-9c93-48b3-a5a0-49d61f061e64","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:44:22.176Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"2ef97afd-9c93-48b3-a5a0-49d61f061e64","Found 1 head Pod":"ray-cluster-head-9lzd6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:44:22.176Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"2ef97afd-9c93-48b3-a5a0-49d61f061e64","head Pod":"ray-cluster-head-9lzd6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-9lzd6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:44:22.176Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"2ef97afd-9c93-48b3-a5a0-49d61f061e64","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-11-03T15:44:22.177Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"2ef97afd-9c93-48b3-a5a0-49d61f061e64","worker Pod":"ray-cluster-small-group-worker-vwxmp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-vwxmp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:44:22.177Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"2ef97afd-9c93-48b3-a5a0-49d61f061e64","worker Pod":"ray-cluster-small-group-worker-hhv2f","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-hhv2f. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:44:22.177Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"2ef97afd-9c93-48b3-a5a0-49d61f061e64","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:44:22.177Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"2ef97afd-9c93-48b3-a5a0-49d61f061e64","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-11-03T15:44:22.177Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"2ef97afd-9c93-48b3-a5a0-49d61f061e64","group":"small-group","indices":{"0":true,"1":true}}
{"level":"info","ts":"2026-11-03T15:44:22.177Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"2ef97afd-9c93-48b3-a5a0-49d61f061e64","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:44:22.177Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"2ef97afd-9c93-48b3-a5a0-49d61f061e64","podName":"ray-cluster-small-group-worker-vwxmp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:44:22.177Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"2ef97afd-9c93-48b3-a5a0-49d61f061e64","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:44:22.177Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"2ef97afd-9c93-48b3-a5a0-49d61f061e64","seconds":300}
{"level":"info","ts":"2026-11-03T15:44:23.189Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"c6e63161-8a05-410b-af7e-afde7c6404c3"}
{"level":"info","ts":"2026-11-03T15:44:23.189Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"c6e63161-8a05-410b-af7e-afde7c6404c3"}
{"level":"info","ts":"2026-11-03T15:44:23.189Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"c6e63161-8a05-410b-af7e-afde7c6404c3","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:44:23.189Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"c6e63161-8a05-410b-af7e-afde7c6404c3","Found 1 head Pod":"ray-cluster-head-9lzd6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:44:23.189Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"c6e63161-8a05-410b-af7e-afde7c6404c3","head Pod":"ray-cluster-head-9lzd6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-9lzd6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:44:23.189Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"c6e63161-8a05-410b-af7e-afde7c6404c3","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-11-03T15:44:23.189Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"c6e63161-8a05-410b-af7e-afde7c6404c3","worker Pod":"ray-cluster-small-group-worker-vwxmp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-vwxmp. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:44:23.189Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"c6e63161-8a05-410b-af7e-afde7c6404c3","worker Pod":"ray-cluster-small-group-worker-hhv2f","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-hhv2f. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:44:23.189Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"c6e63161-8a05-410b-af7e-afde7c6404c3","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:44:23.189Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"c6e63161-8a05-410b-af7e-afde7c6404c3","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-11-03T15:44:23.189Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"c6e63161-8a05-410b-af7e-afde7c6404c3","group":"small-group","indices":{"0":true,"1":true}}
{"level":"info","ts":"2026-11-03T15:44:23.189Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"c6e63161-8a05-410b-af7e-afde7c6404c3","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:44:23.189Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"c6e63161-8a05-410b-af7e-afde7c6404c3","podName":"ray-cluster-small-group-worker-vwxmp","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:42:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:44:23.189Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"c6e63161-8a05-410b-af7e-afde7c6404c3","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:43:00Z","stateTransitionTimes":{"ready":"2026-11-03T15:42:45Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.56","serviceIP":"10.134.0.56","podName":"ray-cluster-head-9lzd6","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-11-03T15:40:56Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:42:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:42:45Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3},"new status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:44:23Z","stateTransitionTimes":{"ready":"2026-11-03T15:42:45Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.56","serviceIP":"10.134.0.56","podName":"ray-cluster-head-9lzd6","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-11-03T15:40:56Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:42:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:42:45Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspending","message":""}],"availableWorkerReplicas":1,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3}}
{"level":"info","ts":"2026-11-03T15:44:23.251Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"9aa3d1ab-513e-4b28-876d-1c1fc59ab198","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:44:25.251Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"59e294c8-39c6-4009-97b1-81e38df0a125"}
{"level":"info","ts":"2026-11-03T15:44:25.251Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"59e294c8-39c6-4009-97b1-81e38df0a125"}
{"level":"info","ts":"2026-11-03T15:44:25.251Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"59e294c8-39c6-4009-97b1-81e38df0a125","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:44:25.251Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"59e294c8-39c6-4009-97b1-81e38df0a125","Found 1 head Pod":"ray-cluster-head-9lzd6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:44:25.251Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"59e294c8-39c6-4009-97b1-81e38df0a125","head Pod":"ray-cluster-head-9lzd6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-9lzd6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:44:25.251Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"59e294c8-39c6-4009-97b1-81e38df0a125","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-11-03T15:44:25.251Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"59e294c8-39c6-4009-97b1-81e38df0a125","worker Pod":"ray-cluster-small-group-worker-vwxmp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-vwxmp. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:44:25.251Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"59e294c8-39c6-4009-97b1-81e38df0a125","worker Pod":"ray-cluster-small-group-worker-hhv2f","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-hhv2f. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:44:25.251Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"59e294c8-39c6-4009-97b1-81e38df0a125","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:44:25.251Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"59e294c8-39c6-4009-97b1-81e38df0a125","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-11-03T15:44:25.251Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"59e294c8-39c6-4009-97b1-81e38df0a125","group":"small-group","indices":{"0":true,"1":true}}
{"level":"info","ts":"2026-11-03T15:44:25.251Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"59e294c8-39c6-4009-97b1-81e38df0a125","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:44:25.252Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"59e294c8-39c6-4009-97b1-81e38df0a125","podName":"ray-cluster-small-group-worker-vwxmp","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:42:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:44:25.252Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"59e294c8-39c6-4009-97b1-81e38df0a125","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:44:25.252Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"59e294c8-39c6-4009-97b1-81e38df0a125","seconds":300}
{"level":"info","ts":"2026-11-03T15:44:28.214Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"553c141f-2ad0-44e7-a6ea-bfe296cd89d6"}
{"level":"info","ts":"2026-11-03T15:44:28.214Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"553c141f-2ad0-44e7-a6ea-bfe296cd89d6"}
{"level":"info","ts":"2026-11-03T15:44:28.214Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"553c141f-2ad0-44e7-a6ea-bfe296cd89d6","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:44:28.214Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"553c141f-2ad0-44e7-a6ea-bfe296cd89d6","Found 1 head Pod":"ray-cluster-head-9lzd6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:44:28.214Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"553c141f-2ad0-44e7-a6ea-bfe296cd89d6","head Pod":"ray-cluster-head-9lzd6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-9lzd6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:44:28.214Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"553c141f-2ad0-44e7-a6ea-bfe296cd89d6","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-11-03T15:44:28.214Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"553c141f-2ad0-44e7-a6ea-bfe296cd89d6","worker Pod":"ray-cluster-small-group-worker-vwxmp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-vwxmp. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:44:28.214Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"553c141f-2ad0-44e7-a6ea-bfe296cd89d6","worker Pod":"ray-cluster-small-group-worker-hhv2f","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-hhv2f. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:44:28.214Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"553c141f-2ad0-44e7-a6ea-bfe296cd89d6","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:44:28.214Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"553c141f-2ad0-44e7-a6ea-bfe296cd89d6","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-11-03T15:44:28.214Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"553c141f-2ad0-44e7-a6ea-bfe296cd89d6","group":"small-group","indices":{"0":true,"1":true}}
{"level":"info","ts":"2026-11-03T15:44:28.214Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"553c141f-2ad0-44e7-a6ea-bfe296cd89d6","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:44:28.214Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"553c141f-2ad0-44e7-a6ea-bfe296cd89d6","podName":"ray-cluster-small-group-worker-vwxmp","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:42:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:44:28.214Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"553c141f-2ad0-44e7-a6ea-bfe296cd89d6","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:44:28.214Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"553c141f-2ad0-44e7-a6ea-bfe296cd89d6","seconds":300}
{"level":"info","ts":"2026-11-03T15:44:29.217Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"3b85b9e8-bbf6-4c8b-ab46-a25f050383f2"}
{"level":"info","ts":"2026-11-03T15:44:29.217Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"3b85b9e8-bbf6-4c8b-ab46-a25f050383f2"}
{"level":"info","ts":"2026-11-03T15:44:29.217Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"3b85b9e8-bbf6-4c8b-ab46-a25f050383f2","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:44:29.217Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"3b85b9e8-bbf6-4c8b-ab46-a25f050383f2","Found 1 head Pod":"ray-cluster-head-9lzd6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:44:29.217Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"3b85b9e8-bbf6-4c8b-ab46-a25f050383f2","head Pod":"ray-cluster-head-9lzd6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-9lzd6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:44:29.217Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"3b85b9e8-bbf6-4c8b-ab46-a25f050383f2","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-11-03T15:44:29.217Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"3b85b9e8-bbf6-4c8b-ab46-a25f050383f2","worker Pod":"ray-cluster-small-group-worker-vwxmp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-vwxmp. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:44:29.217Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"3b85b9e8-bbf6-4c8b-ab46-a25f050383f2","worker Pod":"ray-cluster-small-group-worker-hhv2f","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-hhv2f. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:44:29.217Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"3b85b9e8-bbf6-4c8b-ab46-a25f050383f2","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:44:29.217Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"3b85b9e8-bbf6-4c8b-ab46-a25f050383f2","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-11-03T15:44:29.217Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"3b85b9e8-bbf6-4c8b-ab46-a25f050383f2","group":"small-group","indices":{"0":true,"1":true}}
{"level":"info","ts":"2026-11-03T15:44:29.217Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"3b85b9e8-bbf6-4c8b-ab46-a25f050383f2","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:44:29.217Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"3b85b9e8-bbf6-4c8b-ab46-a25f050383f2","podName":"ray-cluster-small-group-worker-vwxmp","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:42:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:44:29.218Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"3b85b9e8-bbf6-4c8b-ab46-a25f050383f2","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:44:23Z","stateTransitionTimes":{"ready":"2026-11-03T15:42:45Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.56","serviceIP":"10.134.0.56","podName":"ray-cluster-head-9lzd6","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-11-03T15:40:56Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:42:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:42:45Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspending","message":""}],"availableWorkerReplicas":1,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3},"new status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:44:29Z","stateTransitionTimes":{"ready":"2026-11-03T15:42:45Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.56","serviceIP":"10.134.0.56","podName":"ray-cluster-head-9lzd6","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-11-03T15:40:56Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:42:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:42:45Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspending","message":""}],"availableWorkerReplicas":2,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3}}
{"level":"info","ts":"2026-11-03T15:44:29.278Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"44557095-7690-4534-9a9d-d86b46cddce0","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:44:31.279Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"8007ebf3-c111-4901-acde-9b1d174299a2"}
{"level":"info","ts":"2026-11-03T15:44:31.279Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"8007ebf3-c111-4901-acde-9b1d174299a2"}
{"level":"info","ts":"2026-11-03T15:44:31.279Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"8007ebf3-c111-4901-acde-9b1d174299a2","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:44:31.279Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"8007ebf3-c111-4901-acde-9b1d174299a2","Found 1 head Pod":"ray-cluster-head-9lzd6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:44:31.279Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"8007ebf3-c111-4901-acde-9b1d174299a2","head Pod":"ray-cluster-head-9lzd6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-9lzd6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:44:31.279Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"8007ebf3-c111-4901-acde-9b1d174299a2","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-11-03T15:44:31.279Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"8007ebf3-c111-4901-acde-9b1d174299a2","worker Pod":"ray-cluster-small-group-worker-vwxmp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-vwxmp. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:44:31.279Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"8007ebf3-c111-4901-acde-9b1d174299a2","worker Pod":"ray-cluster-small-group-worker-hhv2f","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-hhv2f. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:44:31.279Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"8007ebf3-c111-4901-acde-9b1d174299a2","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:44:31.279Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"8007ebf3-c111-4901-acde-9b1d174299a2","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-11-03T15:44:31.279Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"8007ebf3-c111-4901-acde-9b1d174299a2","group":"small-group","indices":{"0":true,"1":true}}
{"level":"info","ts":"2026-11-03T15:44:31.279Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"8007ebf3-c111-4901-acde-9b1d174299a2","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:44:31.280Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"8007ebf3-c111-4901-acde-9b1d174299a2","podName":"ray-cluster-small-group-worker-vwxmp","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:42:53Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:44:31.280Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"8007ebf3-c111-4901-acde-9b1d174299a2","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:44:31.280Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"8007ebf3-c111-4901-acde-9b1d174299a2","seconds":300}
{"level":"info","ts":"2026-11-03T15:44:34.199Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"910af9a9-d8e2-4bb7-b3a7-dc385ce4f985"}
{"level":"info","ts":"2026-11-03T15:44:34.199Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"910af9a9-d8e2-4bb7-b3a7-dc385ce4f985"}
{"level":"info","ts":"2026-11-03T15:44:34.199Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"910af9a9-d8e2-4bb7-b3a7-dc385ce4f985","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:44:34.199Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"910af9a9-d8e2-4bb7-b3a7-dc385ce4f985","Found 1 head Pod":"ray-cluster-head-9lzd6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:44:34.199Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"910af9a9-d8e2-4bb7-b3a7-dc385ce4f985","head Pod":"ray-cluster-head-9lzd6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-9lzd6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:44:34.199Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"910af9a9-d8e2-4bb7-b3a7-dc385ce4f985","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-11-03T15:44:34.199Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"910af9a9-d8e2-4bb7-b3a7-dc385ce4f985","worker Pod":"ray-cluster-small-group-worker-vwxmp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-vwxmp. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:44:34.199Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"910af9a9-d8e2-4bb7-b3a7-dc385ce4f985","worker Pod":"ray-cluster-small-group-worker-hhv2f","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-hhv2f. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:44:34.199Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"910af9a9-d8e2-4bb7-b3a7-dc385ce4f985","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:44:34.199Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"910af9a9-d8e2-4bb7-b3a7-dc385ce4f985","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-11-03T15:44:34.199Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"910af9a9-d8e2-4bb7-b3a7-dc385ce4f985","group":"small-group","indices":{"0":true,"1":true}}
{"level":"info","ts":"2026-11-03T15:44:34.199Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"910af9a9-d8e2-4bb7-b3a7-dc385ce4f985","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:44:34.199Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"910af9a9-d8e2-4bb7-b3a7-dc385ce4f985","podName":"ray-cluster-small-group-worker-hhv2f","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:43:00Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:44:34.199Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"910af9a9-d8e2-4bb7-b3a7-dc385ce4f985","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:44:29Z","stateTransitionTimes":{"ready":"2026-11-03T15:42:45Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.56","serviceIP":"10.134.0.56","podName":"ray-cluster-head-9lzd6","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-11-03T15:40:56Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:42:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:42:45Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspending","message":""}],"availableWorkerReplicas":2,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3},"new status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:44:34Z","stateTransitionTimes":{"ready":"2026-11-03T15:42:45Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.56","serviceIP":"10.134.0.56","podName":"ray-cluster-head-9lzd6","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-11-03T15:40:56Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:42:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:42:45Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":2,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3}}
{"level":"info","ts":"2026-11-03T15:44:34.260Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"4d7902dd-b6bb-4c8e-ac4b-c007d7d5aa62","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:44:36.261Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"4110fd2e-8f5b-41d3-91ff-83a1250b7542"}
{"level":"info","ts":"2026-11-03T15:44:36.261Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"4110fd2e-8f5b-41d3-91ff-83a1250b7542"}
{"level":"info","ts":"2026-11-03T15:44:36.261Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"4110fd2e-8f5b-41d3-91ff-83a1250b7542","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:44:36.261Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"4110fd2e-8f5b-41d3-91ff-83a1250b7542","Found 1 head Pod":"ray-cluster-head-9lzd6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:44:36.261Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"4110fd2e-8f5b-41d3-91ff-83a1250b7542","head Pod":"ray-cluster-head-9lzd6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-9lzd6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:44:36.261Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"4110fd2e-8f5b-41d3-91ff-83a1250b7542","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-11-03T15:44:36.261Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"4110fd2e-8f5b-41d3-91ff-83a1250b7542","worker Pod":"ray-cluster-small-group-worker-vwxmp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-vwxmp. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:44:36.261Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"4110fd2e-8f5b-41d3-91ff-83a1250b7542","worker Pod":"ray-cluster-small-group-worker-hhv2f","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-hhv2f. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:44:36.261Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"4110fd2e-8f5b-41d3-91ff-83a1250b7542","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:44:36.261Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"4110fd2e-8f5b-41d3-91ff-83a1250b7542","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-11-03T15:44:36.261Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"4110fd2e-8f5b-41d3-91ff-83a1250b7542","group":"small-group","indices":{"0":true,"1":true}}
{"level":"info","ts":"2026-11-03T15:44:36.261Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"4110fd2e-8f5b-41d3-91ff-83a1250b7542","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:44:36.261Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"4110fd2e-8f5b-41d3-91ff-83a1250b7542","podName":"ray-cluster-small-group-worker-hhv2f","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:43:00Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:44:36.262Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"4110fd2e-8f5b-41d3-91ff-83a1250b7542","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:44:36.262Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"4110fd2e-8f5b-41d3-91ff-83a1250b7542","seconds":300}
{"level":"info","ts":"2026-11-03T15:44:40.224Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"8db20a22-841c-4eb9-ba13-d019af9a3dc7"}
{"level":"info","ts":"2026-11-03T15:44:40.224Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"8db20a22-841c-4eb9-ba13-d019af9a3dc7"}
{"level":"info","ts":"2026-11-03T15:44:40.224Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"8db20a22-841c-4eb9-ba13-d019af9a3dc7","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:44:40.225Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"8db20a22-841c-4eb9-ba13-d019af9a3dc7","Found 1 head Pod":"ray-cluster-head-9lzd6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:44:40.225Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"8db20a22-841c-4eb9-ba13-d019af9a3dc7","head Pod":"ray-cluster-head-9lzd6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-9lzd6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:44:40.225Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"8db20a22-841c-4eb9-ba13-d019af9a3dc7","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-11-03T15:44:40.225Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"8db20a22-841c-4eb9-ba13-d019af9a3dc7","worker Pod":"ray-cluster-small-group-worker-hhv2f","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-hhv2f. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:44:40.225Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"8db20a22-841c-4eb9-ba13-d019af9a3dc7","worker Pod":"ray-cluster-small-group-worker-vwxmp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-vwxmp. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:44:40.225Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"8db20a22-841c-4eb9-ba13-d019af9a3dc7","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:44:40.225Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"8db20a22-841c-4eb9-ba13-d019af9a3dc7","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-11-03T15:44:40.225Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"8db20a22-841c-4eb9-ba13-d019af9a3dc7","group":"small-group","indices":{"0":true,"1":true}}
{"level":"info","ts":"2026-11-03T15:44:40.225Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"8db20a22-841c-4eb9-ba13-d019af9a3dc7","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:44:40.225Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"8db20a22-841c-4eb9-ba13-d019af9a3dc7","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:44:34Z","stateTransitionTimes":{"ready":"2026-11-03T15:42:45Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.56","serviceIP":"10.134.0.56","podName":"ray-cluster-head-9lzd6","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-11-03T15:40:56Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:42:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:42:45Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":2,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3},"new status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:44:40Z","stateTransitionTimes":{"ready":"2026-11-03T15:42:45Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.56","serviceIP":"10.134.0.56","podName":"ray-cluster-head-9lzd6","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-11-03T15:40:56Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:42:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:42:45Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":2,"availableWorkerReplicas":2,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3}}
{"level":"info","ts":"2026-11-03T15:44:40.286Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"318a0e81-3aff-41d1-87a6-61d32223e426","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:44:42.287Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"6382f020-f6c4-478a-9979-101a6028b7b8"}
{"level":"info","ts":"2026-11-03T15:44:42.287Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"6382f020-f6c4-478a-9979-101a6028b7b8"}
{"level":"info","ts":"2026-11-03T15:44:42.287Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"6382f020-f6c4-478a-9979-101a6028b7b8","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:44:42.287Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"6382f020-f6c4-478a-9979-101a6028b7b8","Found 1 head Pod":"ray-cluster-head-9lzd6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:44:42.287Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"6382f020-f6c4-478a-9979-101a6028b7b8","head Pod":"ray-cluster-head-9lzd6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-9lzd6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:44:42.287Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"6382f020-f6c4-478a-9979-101a6028b7b8","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-11-03T15:44:42.287Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"6382f020-f6c4-478a-9979-101a6028b7b8","worker Pod":"ray-cluster-small-group-worker-vwxmp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-vwxmp. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:44:42.287Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"6382f020-f6c4-478a-9979-101a6028b7b8","worker Pod":"ray-cluster-small-group-worker-hhv2f","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-hhv2f. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:44:42.287Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"6382f020-f6c4-478a-9979-101a6028b7b8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:44:42.287Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"6382f020-f6c4-478a-9979-101a6028b7b8","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-11-03T15:44:42.287Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"6382f020-f6c4-478a-9979-101a6028b7b8","group":"small-group","indices":{"0":true,"1":true}}
{"level":"info","ts":"2026-11-03T15:44:42.287Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"6382f020-f6c4-478a-9979-101a6028b7b8","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:44:42.287Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"6382f020-f6c4-478a-9979-101a6028b7b8","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:44:42.287Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"6382f020-f6c4-478a-9979-101a6028b7b8","seconds":300}
{"level":"info","ts":"2026-11-03T15:45:38.701Z","logger":"raycluster-resource","msg":"default","name":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:45:38.701Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"ray-cluster","namespace":"test-ns-4jsqd"}
{"level":"info","ts":"2026-11-03T15:45:38.879Z","logger":"raycluster-resource","msg":"validate update","name":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:45:38.934Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"namespace":"test-ns-4jsqd","name":"ray-cluster","reconcileID":"a7923bb8-7ef0-418b-8a76-51c1aa0e3607","namespacedName":{"name":"ray-cluster","namespace":"test-ns-4jsqd"}}
{"level":"info","ts":"2026-11-03T15:45:38.934Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"namespace":"test-ns-4jsqd","name":"ray-cluster","reconcileID":"a7923bb8-7ef0-418b-8a76-51c1aa0e3607","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:45:38.934Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"namespace":"test-ns-4jsqd","name":"ray-cluster","reconcileID":"a7923bb8-7ef0-418b-8a76-51c1aa0e3607","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:45:38.934Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"c4131741-7e64-43f7-a62d-1fec87c2caf1"}
{"level":"info","ts":"2026-11-03T15:45:38.934Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"c4131741-7e64-43f7-a62d-1fec87c2caf1"}
{"level":"info","ts":"2026-11-03T15:45:38.934Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"c4131741-7e64-43f7-a62d-1fec87c2caf1","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:45:38.934Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"c4131741-7e64-43f7-a62d-1fec87c2caf1","Found 1 head Pod":"ray-cluster-head-9lzd6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:45:38.934Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"c4131741-7e64-43f7-a62d-1fec87c2caf1","head Pod":"ray-cluster-head-9lzd6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-9lzd6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:45:38.934Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"c4131741-7e64-43f7-a62d-1fec87c2caf1","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":1}
{"level":"info","ts":"2026-11-03T15:45:38.934Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"c4131741-7e64-43f7-a62d-1fec87c2caf1","worker Pod":"ray-cluster-small-group-worker-vwxmp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-vwxmp. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:45:38.934Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"c4131741-7e64-43f7-a62d-1fec87c2caf1","worker Pod":"ray-cluster-small-group-worker-hhv2f","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-hhv2f. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:45:38.934Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"c4131741-7e64-43f7-a62d-1fec87c2caf1","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:45:38.934Z","logger":"controllers.RayCluster","msg":"Deleting pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"c4131741-7e64-43f7-a62d-1fec87c2caf1","namespace":"test-ns-4jsqd","name":"ray-cluster-small-group-worker-hhv2f"}
{"level":"info","ts":"2026-11-03T15:45:38.934Z","logger":"controllers.NetworkPolicy","msg":"Reconciling NetworkPolicies for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"326b6dcd-dab9-4a0c-ab15-93f255a6bea5","cluster":"ray-cluster","namespace":"test-ns-4jsqd"}
{"level":"info","ts":"2026-11-03T15:45:38.935Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"dea39d22-de06-469b-9bb2-08ed32bde963","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:45:39.037Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"c4131741-7e64-43f7-a62d-1fec87c2caf1","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:45:39.037Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"c4131741-7e64-43f7-a62d-1fec87c2caf1","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:45:39.037Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"c4131741-7e64-43f7-a62d-1fec87c2caf1","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:45:39.037Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"c4131741-7e64-43f7-a62d-1fec87c2caf1","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:44:40Z","stateTransitionTimes":{"ready":"2026-11-03T15:42:45Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.56","serviceIP":"10.134.0.56","podName":"ray-cluster-head-9lzd6","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-11-03T15:40:56Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:42:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:42:45Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":2,"availableWorkerReplicas":2,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3},"new status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:45:39Z","stateTransitionTimes":{"ready":"2026-11-03T15:42:45Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.56","serviceIP":"10.134.0.56","podName":"ray-cluster-head-9lzd6","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-11-03T15:40:56Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:42:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:42:45Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":2,"availableWorkerReplicas":2,"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":4}}
{"level":"info","ts":"2026-11-03T15:45:39.084Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"namespace":"test-ns-4jsqd","name":"ray-cluster","reconcileID":"a7923bb8-7ef0-418b-8a76-51c1aa0e3607","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:45:39.094Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"c4131741-7e64-43f7-a62d-1fec87c2caf1","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-4jsqd","name":"ray-cluster"}}
{"level":"info","ts":"2026-11-03T15:45:39.094Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"c4131741-7e64-43f7-a62d-1fec87c2caf1"}
{"level":"error","ts":"2026-11-03T15:45:39.094Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"c4131741-7e64-43f7-a62d-1fec87c2caf1","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:45:39.094Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"a4783421-0904-4a59-a5bb-290f7d73012f"}
{"level":"info","ts":"2026-11-03T15:45:39.094Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"a4783421-0904-4a59-a5bb-290f7d73012f"}
{"level":"info","ts":"2026-11-03T15:45:39.094Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"a4783421-0904-4a59-a5bb-290f7d73012f","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:45:39.094Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"a4783421-0904-4a59-a5bb-290f7d73012f","Found 1 head Pod":"ray-cluster-head-9lzd6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:45:39.094Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"a4783421-0904-4a59-a5bb-290f7d73012f","head Pod":"ray-cluster-head-9lzd6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-9lzd6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:45:39.094Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"a4783421-0904-4a59-a5bb-290f7d73012f","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-11-03T15:45:39.095Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"a4783421-0904-4a59-a5bb-290f7d73012f","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:44:40Z","stateTransitionTimes":{"ready":"2026-11-03T15:42:45Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.56","serviceIP":"10.134.0.56","podName":"ray-cluster-head-9lzd6","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":4,"lastTransitionTime":"2026-11-03T15:40:56Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:42:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:42:45Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":2,"availableWorkerReplicas":2,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3},"new status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:45:39Z","stateTransitionTimes":{"ready":"2026-11-03T15:42:45Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.56","serviceIP":"10.134.0.56","podName":"ray-cluster-head-9lzd6","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":4,"lastTransitionTime":"2026-11-03T15:40:56Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:42:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:42:45Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":2,"availableWorkerReplicas":2,"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":4}}
{"level":"info","ts":"2026-11-03T15:45:39.139Z","logger":"controllers.NetworkPolicy","msg":"Successfully reconciled NetworkPolicies for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"326b6dcd-dab9-4a0c-ab15-93f255a6bea5","cluster":"ray-cluster","namespace":"test-ns-4jsqd"}
{"level":"info","ts":"2026-11-03T15:45:39.155Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"ac46cef8-15dd-4fc4-a99a-e2e21c405af1"}
{"level":"info","ts":"2026-11-03T15:45:39.155Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"ac46cef8-15dd-4fc4-a99a-e2e21c405af1"}
{"level":"info","ts":"2026-11-03T15:45:39.155Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"ac46cef8-15dd-4fc4-a99a-e2e21c405af1","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:45:39.156Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"ac46cef8-15dd-4fc4-a99a-e2e21c405af1","Found 1 head Pod":"ray-cluster-head-9lzd6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:45:39.156Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"ac46cef8-15dd-4fc4-a99a-e2e21c405af1","head Pod":"ray-cluster-head-9lzd6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-9lzd6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:45:39.156Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"ac46cef8-15dd-4fc4-a99a-e2e21c405af1","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-11-03T15:45:39.156Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"ac46cef8-15dd-4fc4-a99a-e2e21c405af1","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:44:40Z","stateTransitionTimes":{"ready":"2026-11-03T15:42:45Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.56","serviceIP":"10.134.0.56","podName":"ray-cluster-head-9lzd6","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":4,"lastTransitionTime":"2026-11-03T15:40:56Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:42:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:42:45Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":2,"availableWorkerReplicas":2,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3},"new status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:45:39Z","stateTransitionTimes":{"ready":"2026-11-03T15:42:45Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.56","serviceIP":"10.134.0.56","podName":"ray-cluster-head-9lzd6","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":4,"lastTransitionTime":"2026-11-03T15:40:56Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:42:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:42:45Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":2,"availableWorkerReplicas":2,"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":4}}
{"level":"info","ts":"2026-11-03T15:45:39.211Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"ac46cef8-15dd-4fc4-a99a-e2e21c405af1","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-4jsqd","name":"ray-cluster"}}
{"level":"info","ts":"2026-11-03T15:45:39.211Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"ac46cef8-15dd-4fc4-a99a-e2e21c405af1"}
{"level":"error","ts":"2026-11-03T15:45:39.211Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"ac46cef8-15dd-4fc4-a99a-e2e21c405af1","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:45:39.217Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"376c5329-454b-4713-9c77-b2f6c4b291b9"}
{"level":"info","ts":"2026-11-03T15:45:39.217Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"376c5329-454b-4713-9c77-b2f6c4b291b9"}
{"level":"info","ts":"2026-11-03T15:45:39.217Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"376c5329-454b-4713-9c77-b2f6c4b291b9","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:45:39.217Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"376c5329-454b-4713-9c77-b2f6c4b291b9","Found 1 head Pod":"ray-cluster-head-9lzd6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:45:39.217Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"376c5329-454b-4713-9c77-b2f6c4b291b9","head Pod":"ray-cluster-head-9lzd6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-9lzd6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:45:39.217Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"376c5329-454b-4713-9c77-b2f6c4b291b9","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-11-03T15:45:39.217Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"376c5329-454b-4713-9c77-b2f6c4b291b9","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:45:39.217Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"376c5329-454b-4713-9c77-b2f6c4b291b9","seconds":300}
{"level":"info","ts":"2026-11-03T15:45:39.488Z","logger":"controllers.RayClusterMTLS","msg":"Updating worker certificate","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"dea39d22-de06-469b-9bb2-08ed32bde963","certificate":"ray-worker-cert-ray-cluster","ipAddresses":["10.132.0.52","127.0.0.1"]}
{"level":"info","ts":"2026-11-03T15:45:39.560Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"7a9d5202-9113-4e83-b9f2-8835856b49bf"}
{"level":"info","ts":"2026-11-03T15:45:39.560Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"7a9d5202-9113-4e83-b9f2-8835856b49bf"}
{"level":"info","ts":"2026-11-03T15:45:39.561Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"7a9d5202-9113-4e83-b9f2-8835856b49bf","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:45:39.561Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"7a9d5202-9113-4e83-b9f2-8835856b49bf","Found 1 head Pod":"ray-cluster-head-9lzd6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:45:39.561Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"7a9d5202-9113-4e83-b9f2-8835856b49bf","head Pod":"ray-cluster-head-9lzd6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-9lzd6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:45:39.561Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"7a9d5202-9113-4e83-b9f2-8835856b49bf","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-11-03T15:45:39.561Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"7a9d5202-9113-4e83-b9f2-8835856b49bf","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:45:39Z","stateTransitionTimes":{"ready":"2026-11-03T15:42:45Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.56","serviceIP":"10.134.0.56","podName":"ray-cluster-head-9lzd6","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":4,"lastTransitionTime":"2026-11-03T15:40:56Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:42:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:42:45Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":2,"availableWorkerReplicas":2,"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":4},"new status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:45:39Z","stateTransitionTimes":{"ready":"2026-11-03T15:42:45Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.56","serviceIP":"10.134.0.56","podName":"ray-cluster-head-9lzd6","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":4,"lastTransitionTime":"2026-11-03T15:40:56Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:42:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:42:45Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":4}}
{"level":"info","ts":"2026-11-03T15:45:39.921Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"dea39d22-de06-469b-9bb2-08ed32bde963","certificate":"ray-worker-cert-ray-cluster"}
{"level":"info","ts":"2026-11-03T15:45:39.921Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"dea39d22-de06-469b-9bb2-08ed32bde963"}
{"level":"info","ts":"2026-11-03T15:45:39.921Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"5a10ce0f-9c32-4c64-b693-dd31571cb4c3","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:45:40.563Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"e12e2b9f-66a4-414f-ad91-d18944a9d066"}
{"level":"info","ts":"2026-11-03T15:45:40.563Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"e12e2b9f-66a4-414f-ad91-d18944a9d066"}
{"level":"info","ts":"2026-11-03T15:45:40.563Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"e12e2b9f-66a4-414f-ad91-d18944a9d066","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:45:40.563Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"e12e2b9f-66a4-414f-ad91-d18944a9d066","Found 1 head Pod":"ray-cluster-head-9lzd6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:45:40.563Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"e12e2b9f-66a4-414f-ad91-d18944a9d066","head Pod":"ray-cluster-head-9lzd6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-9lzd6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:45:40.563Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"e12e2b9f-66a4-414f-ad91-d18944a9d066","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-11-03T15:45:40.563Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"e12e2b9f-66a4-414f-ad91-d18944a9d066","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:45:40.563Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"e12e2b9f-66a4-414f-ad91-d18944a9d066","seconds":300}
{"level":"info","ts":"2026-11-03T15:45:40.566Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"67dfa495-afa2-4e15-8a63-c4d8398a3384"}
{"level":"info","ts":"2026-11-03T15:45:40.566Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"67dfa495-afa2-4e15-8a63-c4d8398a3384"}
{"level":"info","ts":"2026-11-03T15:45:40.566Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"67dfa495-afa2-4e15-8a63-c4d8398a3384","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:45:40.567Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"67dfa495-afa2-4e15-8a63-c4d8398a3384","Found 1 head Pod":"ray-cluster-head-9lzd6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:45:40.567Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"67dfa495-afa2-4e15-8a63-c4d8398a3384","head Pod":"ray-cluster-head-9lzd6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-9lzd6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:45:40.567Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"67dfa495-afa2-4e15-8a63-c4d8398a3384","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":1}
{"level":"info","ts":"2026-11-03T15:45:40.567Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"67dfa495-afa2-4e15-8a63-c4d8398a3384","worker Pod":"ray-cluster-small-group-worker-vwxmp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-vwxmp. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:45:40.567Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"67dfa495-afa2-4e15-8a63-c4d8398a3384","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:45:40.567Z","logger":"controllers.RayCluster","msg":"Deleting pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"67dfa495-afa2-4e15-8a63-c4d8398a3384","namespace":"test-ns-4jsqd","name":"ray-cluster-small-group-worker-hhv2f"}
{"level":"info","ts":"2026-11-03T15:45:40.577Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"5a10ce0f-9c32-4c64-b693-dd31571cb4c3","certificate":"ray-worker-cert-ray-cluster"}
{"level":"info","ts":"2026-11-03T15:45:40.577Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"5a10ce0f-9c32-4c64-b693-dd31571cb4c3"}
{"level":"info","ts":"2026-11-03T15:45:40.616Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"67dfa495-afa2-4e15-8a63-c4d8398a3384","The worker Pod has already been deleted":"ray-cluster-small-group-worker-hhv2f"}
{"level":"info","ts":"2026-11-03T15:45:40.616Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"67dfa495-afa2-4e15-8a63-c4d8398a3384","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:45:40.616Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"67dfa495-afa2-4e15-8a63-c4d8398a3384","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:45:40.616Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"67dfa495-afa2-4e15-8a63-c4d8398a3384","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:45:40.617Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"67dfa495-afa2-4e15-8a63-c4d8398a3384","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:45:40.617Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"67dfa495-afa2-4e15-8a63-c4d8398a3384","seconds":300}
{"level":"info","ts":"2026-11-03T15:46:10.578Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"93065f3e-90dc-441f-b2cf-ba8b4f18d9a0","cluster":"ray-cluster"}
{"level":"error","ts":"2026-11-03T15:46:45.429Z","logger":"events","msg":"Server rejected event (will not retry!)","event":{"namespace":"test-ns-kbx4w","name":"ttl-initializing-vl6x2.18e493233f768a40"},"error":"namespaces \"test-ns-kbx4w\" not found","stacktrace":"k8s.io/client-go/tools/events.recordEvent\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:270\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).finishSeries\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:153\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext.func1\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:255\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:256\nk8s.io/apimachinery/pkg/util/wait.JitterUntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:223\nk8s.io/apimachinery/pkg/util/wait.UntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:172"}
{"level":"error","ts":"2026-11-03T15:46:45.581Z","logger":"events","msg":"Server rejected event (will not retry!)","event":{"namespace":"test-ns-kbx4w","name":"counter-auth-qmw8r.18e49303d2c43c14"},"error":"namespaces \"test-ns-kbx4w\" not found","stacktrace":"k8s.io/client-go/tools/events.recordEvent\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:270\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).finishSeries\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:153\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext.func1\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:255\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:256\nk8s.io/apimachinery/pkg/util/wait.JitterUntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:223\nk8s.io/apimachinery/pkg/util/wait.UntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:172"}
{"level":"error","ts":"2026-11-03T15:46:45.732Z","logger":"events","msg":"Server rejected event (will not retry!)","event":{"namespace":"test-ns-kbx4w","name":"counter-auth-qmw8r.18e49303b9b0dc9d"},"error":"namespaces \"test-ns-kbx4w\" not found","stacktrace":"k8s.io/client-go/tools/events.recordEvent\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:270\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).finishSeries\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:153\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext.func1\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:255\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:256\nk8s.io/apimachinery/pkg/util/wait.JitterUntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:223\nk8s.io/apimachinery/pkg/util/wait.UntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:172"}
{"level":"error","ts":"2026-11-03T15:46:45.883Z","logger":"events","msg":"Server rejected event (will not retry!)","event":{"namespace":"test-ns-kbx4w","name":"ttl-waiting-kjf64.18e49324ed417649"},"error":"namespaces \"test-ns-kbx4w\" not found","stacktrace":"k8s.io/client-go/tools/events.recordEvent\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:270\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).finishSeries\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:153\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext.func1\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:255\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:256\nk8s.io/apimachinery/pkg/util/wait.JitterUntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:223\nk8s.io/apimachinery/pkg/util/wait.UntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:172"}
{"level":"error","ts":"2026-11-03T15:46:46.035Z","logger":"events","msg":"Server rejected event (will not retry!)","event":{"namespace":"test-ns-kbx4w","name":"long-running-hvml9.18e492a2a33e9cee"},"error":"namespaces \"test-ns-kbx4w\" not found","stacktrace":"k8s.io/client-go/tools/events.recordEvent\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:270\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).finishSeries\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:153\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext.func1\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:255\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:256\nk8s.io/apimachinery/pkg/util/wait.JitterUntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:223\nk8s.io/apimachinery/pkg/util/wait.UntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:172"}
{"level":"error","ts":"2026-11-03T15:46:46.187Z","logger":"events","msg":"Server rejected event (will not retry!)","event":{"namespace":"test-ns-kbx4w","name":"counter-auth-qmw8r.18e493028bb2d568"},"error":"namespaces \"test-ns-kbx4w\" not found","stacktrace":"k8s.io/client-go/tools/events.recordEvent\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:270\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).finishSeries\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:153\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext.func1\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:255\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:256\nk8s.io/apimachinery/pkg/util/wait.JitterUntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:223\nk8s.io/apimachinery/pkg/util/wait.UntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:172"}
{"level":"error","ts":"2026-11-03T15:46:46.338Z","logger":"events","msg":"Server rejected event (will not retry!)","event":{"namespace":"test-ns-kbx4w","name":"ttl-waiting-kjf64.18e493260a09d4a7"},"error":"namespaces \"test-ns-kbx4w\" not found","stacktrace":"k8s.io/client-go/tools/events.recordEvent\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:270\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).finishSeries\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:153\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext.func1\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:255\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:256\nk8s.io/apimachinery/pkg/util/wait.JitterUntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:223\nk8s.io/apimachinery/pkg/util/wait.UntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:172"}
{"level":"info","ts":"2026-11-03T15:46:49.866Z","logger":"raycluster-resource","msg":"default","name":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:46:49.866Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"ray-cluster","namespace":"test-ns-4jsqd"}
{"level":"info","ts":"2026-11-03T15:46:49.923Z","logger":"raycluster-resource","msg":"validate update","name":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:46:49.979Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"namespace":"test-ns-4jsqd","name":"ray-cluster","reconcileID":"54fafe8b-1b9b-42f9-a564-7d99f3a9b409","namespacedName":{"name":"ray-cluster","namespace":"test-ns-4jsqd"}}
{"level":"info","ts":"2026-11-03T15:46:49.979Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"namespace":"test-ns-4jsqd","name":"ray-cluster","reconcileID":"54fafe8b-1b9b-42f9-a564-7d99f3a9b409","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:46:49.979Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"namespace":"test-ns-4jsqd","name":"ray-cluster","reconcileID":"54fafe8b-1b9b-42f9-a564-7d99f3a9b409","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:46:49.979Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"27ead300-940f-4d8e-9dde-f3615a8fb723"}
{"level":"info","ts":"2026-11-03T15:46:49.979Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"27ead300-940f-4d8e-9dde-f3615a8fb723"}
{"level":"info","ts":"2026-11-03T15:46:49.979Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"27ead300-940f-4d8e-9dde-f3615a8fb723","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:46:49.979Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"27ead300-940f-4d8e-9dde-f3615a8fb723","Found 1 head Pod":"ray-cluster-head-9lzd6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:46:49.979Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"27ead300-940f-4d8e-9dde-f3615a8fb723","head Pod":"ray-cluster-head-9lzd6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-9lzd6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:46:49.979Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"27ead300-940f-4d8e-9dde-f3615a8fb723","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-11-03T15:46:49.979Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"27ead300-940f-4d8e-9dde-f3615a8fb723","worker Pod":"ray-cluster-small-group-worker-vwxmp","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-vwxmp. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:46:49.979Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"27ead300-940f-4d8e-9dde-f3615a8fb723","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:46:49.979Z","logger":"controllers.RayCluster","msg":"Deleting pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"27ead300-940f-4d8e-9dde-f3615a8fb723","namespace":"test-ns-4jsqd","name":"ray-cluster-small-group-worker-vwxmp"}
{"level":"info","ts":"2026-11-03T15:46:49.979Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"83d95e33-7692-4d2b-a42e-2fed8c0e5ef0","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:46:49.979Z","logger":"controllers.NetworkPolicy","msg":"Reconciling NetworkPolicies for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"3cf10f86-05e9-43f7-936f-bfde7d471911","cluster":"ray-cluster","namespace":"test-ns-4jsqd"}
{"level":"info","ts":"2026-11-03T15:46:50.082Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"27ead300-940f-4d8e-9dde-f3615a8fb723","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-11-03T15:46:50.082Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"27ead300-940f-4d8e-9dde-f3615a8fb723","group":"small-group","indices":{}}
{"level":"info","ts":"2026-11-03T15:46:50.082Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"27ead300-940f-4d8e-9dde-f3615a8fb723","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:46:50.082Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"27ead300-940f-4d8e-9dde-f3615a8fb723","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:45:39Z","stateTransitionTimes":{"ready":"2026-11-03T15:42:45Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.56","serviceIP":"10.134.0.56","podName":"ray-cluster-head-9lzd6","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":4,"lastTransitionTime":"2026-11-03T15:40:56Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:42:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:42:45Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":4},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:46:50Z","stateTransitionTimes":{"ready":"2026-11-03T15:42:45Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.56","serviceIP":"10.134.0.56","podName":"ray-cluster-head-9lzd6","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":4,"lastTransitionTime":"2026-11-03T15:40:56Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:42:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:42:45Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":5}}
{"level":"info","ts":"2026-11-03T15:46:50.125Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"namespace":"test-ns-4jsqd","name":"ray-cluster","reconcileID":"54fafe8b-1b9b-42f9-a564-7d99f3a9b409","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:46:50.135Z","logger":"controllers.NetworkPolicy","msg":"Successfully reconciled NetworkPolicies for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"3cf10f86-05e9-43f7-936f-bfde7d471911","cluster":"ray-cluster","namespace":"test-ns-4jsqd"}
{"level":"info","ts":"2026-11-03T15:46:50.136Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"27ead300-940f-4d8e-9dde-f3615a8fb723","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-4jsqd","name":"ray-cluster"}}
{"level":"info","ts":"2026-11-03T15:46:50.136Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"27ead300-940f-4d8e-9dde-f3615a8fb723"}
{"level":"error","ts":"2026-11-03T15:46:50.136Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"27ead300-940f-4d8e-9dde-f3615a8fb723","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:46:50.136Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"218a82e0-0d32-4f33-a866-5b886f39ae1f"}
{"level":"info","ts":"2026-11-03T15:46:50.136Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"218a82e0-0d32-4f33-a866-5b886f39ae1f"}
{"level":"info","ts":"2026-11-03T15:46:50.136Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"218a82e0-0d32-4f33-a866-5b886f39ae1f","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:46:50.137Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"218a82e0-0d32-4f33-a866-5b886f39ae1f","Found 1 head Pod":"ray-cluster-head-9lzd6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:46:50.137Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"218a82e0-0d32-4f33-a866-5b886f39ae1f","head Pod":"ray-cluster-head-9lzd6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-9lzd6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:46:50.137Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"218a82e0-0d32-4f33-a866-5b886f39ae1f","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-11-03T15:46:50.137Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"218a82e0-0d32-4f33-a866-5b886f39ae1f","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:45:39Z","stateTransitionTimes":{"ready":"2026-11-03T15:42:45Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.56","serviceIP":"10.134.0.56","podName":"ray-cluster-head-9lzd6","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-11-03T15:40:56Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:42:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:42:45Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":4},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:46:50Z","stateTransitionTimes":{"ready":"2026-11-03T15:42:45Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.56","serviceIP":"10.134.0.56","podName":"ray-cluster-head-9lzd6","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-11-03T15:40:56Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:42:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:42:45Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":5}}
{"level":"info","ts":"2026-11-03T15:46:50.197Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"fe7a2867-9a76-448d-8436-61750999925e"}
{"level":"info","ts":"2026-11-03T15:46:50.197Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"fe7a2867-9a76-448d-8436-61750999925e"}
{"level":"info","ts":"2026-11-03T15:46:50.197Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"fe7a2867-9a76-448d-8436-61750999925e","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:46:50.198Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"fe7a2867-9a76-448d-8436-61750999925e","Found 1 head Pod":"ray-cluster-head-9lzd6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:46:50.198Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"fe7a2867-9a76-448d-8436-61750999925e","head Pod":"ray-cluster-head-9lzd6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-9lzd6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:46:50.198Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"fe7a2867-9a76-448d-8436-61750999925e","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-11-03T15:46:50.198Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"fe7a2867-9a76-448d-8436-61750999925e","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:45:39Z","stateTransitionTimes":{"ready":"2026-11-03T15:42:45Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.56","serviceIP":"10.134.0.56","podName":"ray-cluster-head-9lzd6","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-11-03T15:40:56Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:42:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:42:45Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":4},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:46:50Z","stateTransitionTimes":{"ready":"2026-11-03T15:42:45Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.56","serviceIP":"10.134.0.56","podName":"ray-cluster-head-9lzd6","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-11-03T15:40:56Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:42:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:42:45Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":5}}
{"level":"info","ts":"2026-11-03T15:46:50.253Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"fe7a2867-9a76-448d-8436-61750999925e","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-4jsqd","name":"ray-cluster"}}
{"level":"info","ts":"2026-11-03T15:46:50.253Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"fe7a2867-9a76-448d-8436-61750999925e"}
{"level":"error","ts":"2026-11-03T15:46:50.253Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"fe7a2867-9a76-448d-8436-61750999925e","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:46:50.258Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"2a2ae386-042f-4cae-a83a-53b3dfd526b5"}
{"level":"info","ts":"2026-11-03T15:46:50.258Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"2a2ae386-042f-4cae-a83a-53b3dfd526b5"}
{"level":"info","ts":"2026-11-03T15:46:50.258Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"2a2ae386-042f-4cae-a83a-53b3dfd526b5","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:46:50.258Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"2a2ae386-042f-4cae-a83a-53b3dfd526b5","Found 1 head Pod":"ray-cluster-head-9lzd6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:46:50.258Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"2a2ae386-042f-4cae-a83a-53b3dfd526b5","head Pod":"ray-cluster-head-9lzd6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-9lzd6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:46:50.258Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"2a2ae386-042f-4cae-a83a-53b3dfd526b5","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-11-03T15:46:50.259Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"2a2ae386-042f-4cae-a83a-53b3dfd526b5","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:46:50.259Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"2a2ae386-042f-4cae-a83a-53b3dfd526b5","seconds":300}
{"level":"info","ts":"2026-11-03T15:46:50.437Z","logger":"controllers.RayClusterMTLS","msg":"Updating worker certificate","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"83d95e33-7692-4d2b-a42e-2fed8c0e5ef0","certificate":"ray-worker-cert-ray-cluster","ipAddresses":["127.0.0.1"]}
{"level":"info","ts":"2026-11-03T15:46:50.601Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"0284dfd7-54ff-4986-97e2-cb7036cd4064"}
{"level":"info","ts":"2026-11-03T15:46:50.601Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"0284dfd7-54ff-4986-97e2-cb7036cd4064"}
{"level":"info","ts":"2026-11-03T15:46:50.601Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"0284dfd7-54ff-4986-97e2-cb7036cd4064","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:46:50.601Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"0284dfd7-54ff-4986-97e2-cb7036cd4064","Found 1 head Pod":"ray-cluster-head-9lzd6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:46:50.601Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"0284dfd7-54ff-4986-97e2-cb7036cd4064","head Pod":"ray-cluster-head-9lzd6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-9lzd6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:46:50.601Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"0284dfd7-54ff-4986-97e2-cb7036cd4064","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-11-03T15:46:50.601Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"0284dfd7-54ff-4986-97e2-cb7036cd4064","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:46:50Z","stateTransitionTimes":{"ready":"2026-11-03T15:42:45Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.56","serviceIP":"10.134.0.56","podName":"ray-cluster-head-9lzd6","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-11-03T15:40:56Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:42:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:42:45Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":5},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:46:50Z","stateTransitionTimes":{"ready":"2026-11-03T15:42:45Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.56","serviceIP":"10.134.0.56","podName":"ray-cluster-head-9lzd6","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-11-03T15:40:56Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:42:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:42:45Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":5}}
{"level":"info","ts":"2026-11-03T15:46:50.747Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"83d95e33-7692-4d2b-a42e-2fed8c0e5ef0","certificate":"ray-worker-cert-ray-cluster"}
{"level":"info","ts":"2026-11-03T15:46:50.747Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"83d95e33-7692-4d2b-a42e-2fed8c0e5ef0"}
{"level":"info","ts":"2026-11-03T15:46:50.748Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"ac7ba45e-00c4-48ce-bb98-aa6072b7be5e","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:46:50.793Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"a1084039-281e-4699-a774-4c608f5bed28"}
{"level":"info","ts":"2026-11-03T15:46:50.793Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"a1084039-281e-4699-a774-4c608f5bed28"}
{"level":"info","ts":"2026-11-03T15:46:50.793Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"a1084039-281e-4699-a774-4c608f5bed28","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:46:50.793Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"a1084039-281e-4699-a774-4c608f5bed28","Found 1 head Pod":"ray-cluster-head-9lzd6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:46:50.793Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"a1084039-281e-4699-a774-4c608f5bed28","head Pod":"ray-cluster-head-9lzd6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-9lzd6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:46:50.793Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"a1084039-281e-4699-a774-4c608f5bed28","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-11-03T15:46:50.793Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"a1084039-281e-4699-a774-4c608f5bed28","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:46:50.793Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"a1084039-281e-4699-a774-4c608f5bed28","seconds":300}
{"level":"info","ts":"2026-11-03T15:46:50.797Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"f418d676-6346-476c-940e-54797285d0c6"}
{"level":"info","ts":"2026-11-03T15:46:50.797Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"f418d676-6346-476c-940e-54797285d0c6"}
{"level":"info","ts":"2026-11-03T15:46:50.797Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"f418d676-6346-476c-940e-54797285d0c6","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:46:50.797Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"f418d676-6346-476c-940e-54797285d0c6","Found 1 head Pod":"ray-cluster-head-9lzd6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:46:50.797Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"f418d676-6346-476c-940e-54797285d0c6","head Pod":"ray-cluster-head-9lzd6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-9lzd6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:46:50.797Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"f418d676-6346-476c-940e-54797285d0c6","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-11-03T15:46:50.797Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"f418d676-6346-476c-940e-54797285d0c6","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:46:50.797Z","logger":"controllers.RayCluster","msg":"Deleting pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"f418d676-6346-476c-940e-54797285d0c6","namespace":"test-ns-4jsqd","name":"ray-cluster-small-group-worker-vwxmp"}
{"level":"info","ts":"2026-11-03T15:46:50.846Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"f418d676-6346-476c-940e-54797285d0c6","The worker Pod has already been deleted":"ray-cluster-small-group-worker-vwxmp"}
{"level":"info","ts":"2026-11-03T15:46:50.846Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"f418d676-6346-476c-940e-54797285d0c6","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-11-03T15:46:50.847Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"f418d676-6346-476c-940e-54797285d0c6","group":"small-group","indices":{}}
{"level":"info","ts":"2026-11-03T15:46:50.847Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"f418d676-6346-476c-940e-54797285d0c6","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:46:50.847Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"f418d676-6346-476c-940e-54797285d0c6","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:46:50.847Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"f418d676-6346-476c-940e-54797285d0c6","seconds":300}
{"level":"info","ts":"2026-11-03T15:46:51.173Z","logger":"raycluster-resource","msg":"default","name":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:46:51.173Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"ray-cluster","namespace":"test-ns-zjpgs"}
{"level":"info","ts":"2026-11-03T15:46:51.229Z","logger":"raycluster-resource","msg":"validate create","name":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:46:51.285Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"namespace":"test-ns-zjpgs","name":"ray-cluster","reconcileID":"9d4ed830-2dc4-465d-8fb9-23f60d3f910a","namespacedName":{"name":"ray-cluster","namespace":"test-ns-zjpgs"}}
{"level":"info","ts":"2026-11-03T15:46:51.285Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"namespace":"test-ns-zjpgs","name":"ray-cluster","reconcileID":"9d4ed830-2dc4-465d-8fb9-23f60d3f910a","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:46:51.285Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"namespace":"test-ns-zjpgs","name":"ray-cluster","reconcileID":"9d4ed830-2dc4-465d-8fb9-23f60d3f910a","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:46:51.285Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"namespace":"test-ns-zjpgs","name":"ray-cluster","reconcileID":"9d4ed830-2dc4-465d-8fb9-23f60d3f910a","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:46:51.286Z","logger":"controllers.NetworkPolicy","msg":"Reconciling NetworkPolicies for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9a8c5bfd-0375-45a0-9495-7755245f4dbe","cluster":"ray-cluster","namespace":"test-ns-zjpgs"}
{"level":"info","ts":"2026-11-03T15:46:51.337Z","logger":"controllers.RayCluster","msg":"Created service account for Ray Autoscaler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"7c88827f-2b37-4992-947a-f7c1c2f44071","name":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:46:51.340Z","logger":"controllers.NetworkPolicy","msg":"Successfully created NetworkPolicy","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9a8c5bfd-0375-45a0-9495-7755245f4dbe","name":"ray-cluster-head"}
{"level":"info","ts":"2026-11-03T15:46:51.343Z","logger":"raycluster-resource","msg":"default","name":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:46:51.343Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"ray-cluster","namespace":"test-ns-zjpgs"}
{"level":"info","ts":"2026-11-03T15:46:51.393Z","logger":"controllers.NetworkPolicy","msg":"Successfully created NetworkPolicy","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9a8c5bfd-0375-45a0-9495-7755245f4dbe","name":"ray-cluster-workers-small-group"}
{"level":"info","ts":"2026-11-03T15:46:51.394Z","logger":"controllers.NetworkPolicy","msg":"Successfully reconciled NetworkPolicies for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9a8c5bfd-0375-45a0-9495-7755245f4dbe","cluster":"ray-cluster","namespace":"test-ns-zjpgs"}
{"level":"info","ts":"2026-11-03T15:46:51.394Z","logger":"controllers.NetworkPolicy","msg":"Reconciling NetworkPolicies for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"dc6b14ff-f654-4053-89b2-f84c1bf336b4","cluster":"ray-cluster","namespace":"test-ns-zjpgs"}
{"level":"info","ts":"2026-11-03T15:46:51.399Z","logger":"raycluster-resource","msg":"validate update","name":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:46:51.402Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"ac7ba45e-00c4-48ce-bb98-aa6072b7be5e","certificate":"ray-worker-cert-ray-cluster"}
{"level":"info","ts":"2026-11-03T15:46:51.402Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"ac7ba45e-00c4-48ce-bb98-aa6072b7be5e"}
{"level":"info","ts":"2026-11-03T15:46:51.402Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"10524022-1126-4bce-8e5c-6054af7b337f","cluster":"ray-cluster"}
{"level":"error","ts":"2026-11-03T15:46:51.451Z","logger":"events","msg":"Server rejected event (will not retry!)","event":{"namespace":"test-ns-4jsqd","name":"ray-cluster.18e493930de3d390"},"error":"events.events.k8s.io \"ray-cluster.18e493930de3d390\" is forbidden: unable to create new content in namespace test-ns-4jsqd because it is being terminated","stacktrace":"k8s.io/client-go/tools/events.recordEvent\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:270\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).attemptRecording\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:211\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).recordToSink.func1\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:200"}
{"level":"info","ts":"2026-11-03T15:46:51.476Z","logger":"controllers.RayCluster","msg":"Created role for Ray Autoscaler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"7c88827f-2b37-4992-947a-f7c1c2f44071","name":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:46:51.508Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"namespace":"test-ns-zjpgs","name":"ray-cluster","reconcileID":"9d4ed830-2dc4-465d-8fb9-23f60d3f910a","name":"ray-cluster-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-11-03T15:46:51.508Z","logger":"authentication-controller","msg":"Autoscaler RoleBinding does not exist yet, will retry auth SA patch","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"namespace":"test-ns-zjpgs","name":"ray-cluster","reconcileID":"9d4ed830-2dc4-465d-8fb9-23f60d3f910a","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:46:51.508Z","logger":"controllers.NetworkPolicy","msg":"Successfully reconciled NetworkPolicies for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"dc6b14ff-f654-4053-89b2-f84c1bf336b4","cluster":"ray-cluster","namespace":"test-ns-zjpgs"}
{"level":"error","ts":"2026-11-03T15:46:51.561Z","logger":"authentication-controller","msg":"Failed to update AuthenticationReady condition","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"namespace":"test-ns-zjpgs","name":"ray-cluster","reconcileID":"9d4ed830-2dc4-465d-8fb9-23f60d3f910a","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleOIDCConfiguration\n\t/workspace/controllers/ray/authentication_controller.go:280\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:155\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:46:51.561Z","logger":"authentication-controller","msg":"Autoscaler RoleBinding not ready yet, requeueing","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"namespace":"test-ns-zjpgs","name":"ray-cluster","reconcileID":"9d4ed830-2dc4-465d-8fb9-23f60d3f910a","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:46:51.561Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"namespace":"test-ns-zjpgs","name":"ray-cluster","reconcileID":"f20c1d89-2f77-4c45-800f-b15e945238e8","namespacedName":{"name":"ray-cluster","namespace":"test-ns-zjpgs"}}
{"level":"info","ts":"2026-11-03T15:46:51.561Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"namespace":"test-ns-zjpgs","name":"ray-cluster","reconcileID":"f20c1d89-2f77-4c45-800f-b15e945238e8","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:46:51.561Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"namespace":"test-ns-zjpgs","name":"ray-cluster","reconcileID":"f20c1d89-2f77-4c45-800f-b15e945238e8","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:46:51.561Z","logger":"authentication-controller","msg":"Autoscaler RoleBinding does not exist yet, will retry auth SA patch","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"namespace":"test-ns-zjpgs","name":"ray-cluster","reconcileID":"f20c1d89-2f77-4c45-800f-b15e945238e8","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:46:51.579Z","logger":"controllers.RayCluster","msg":"Created role binding for Ray Autoscaler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"7c88827f-2b37-4992-947a-f7c1c2f44071","name":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:46:51.579Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"7c88827f-2b37-4992-947a-f7c1c2f44071"}
{"level":"info","ts":"2026-11-03T15:46:51.579Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"7c88827f-2b37-4992-947a-f7c1c2f44071"}
{"level":"info","ts":"2026-11-03T15:46:51.619Z","logger":"authentication-controller","msg":"Autoscaler RoleBinding not ready yet, requeueing","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"namespace":"test-ns-zjpgs","name":"ray-cluster","reconcileID":"f20c1d89-2f77-4c45-800f-b15e945238e8","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:46:51.633Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"namespace":"test-ns-zjpgs","name":"ray-cluster","reconcileID":"062c4941-9241-492d-a9d1-88432e4a82f9","namespacedName":{"name":"ray-cluster","namespace":"test-ns-zjpgs"}}
{"level":"info","ts":"2026-11-03T15:46:51.633Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"namespace":"test-ns-zjpgs","name":"ray-cluster","reconcileID":"062c4941-9241-492d-a9d1-88432e4a82f9","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:46:51.633Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"namespace":"test-ns-zjpgs","name":"ray-cluster","reconcileID":"062c4941-9241-492d-a9d1-88432e4a82f9","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:46:51.633Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"7c88827f-2b37-4992-947a-f7c1c2f44071","name":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:46:51.634Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"7c88827f-2b37-4992-947a-f7c1c2f44071"}
{"level":"info","ts":"2026-11-03T15:46:51.683Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"7c88827f-2b37-4992-947a-f7c1c2f44071","error":"certificate ray-head-cert-ray-cluster not found: certificates.cert-manager.io \"ray-head-cert-ray-cluster\" not found"}
{"level":"error","ts":"2026-11-03T15:46:51.683Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"7c88827f-2b37-4992-947a-f7c1c2f44071","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster not found: certificates.cert-manager.io \"ray-head-cert-ray-cluster\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:46:51.683Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"7c88827f-2b37-4992-947a-f7c1c2f44071","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster not found: certificates.cert-manager.io \"ray-head-cert-ray-cluster\" not found"}
{"level":"info","ts":"2026-11-03T15:46:51.683Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"7c88827f-2b37-4992-947a-f7c1c2f44071","filter labels":["test-ns-zjpgs",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:46:51.683Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"7c88827f-2b37-4992-947a-f7c1c2f44071","filter labels":["test-ns-zjpgs",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:46:51.683Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"7c88827f-2b37-4992-947a-f7c1c2f44071","filter labels":["test-ns-zjpgs",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:46:51.683Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"7c88827f-2b37-4992-947a-f7c1c2f44071","name":"ray-cluster","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:46:51Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:46:51.713Z","logger":"authentication-controller","msg":"Added auth proxy SA to autoscaler RoleBinding","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"namespace":"test-ns-zjpgs","name":"ray-cluster","reconcileID":"062c4941-9241-492d-a9d1-88432e4a82f9","roleBinding":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"authServiceAccount":"ray-cluster-oauth-proxy-sa","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-11-03T15:46:51.737Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"7c88827f-2b37-4992-947a-f7c1c2f44071","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-zjpgs","name":"ray-cluster"}}
{"level":"info","ts":"2026-11-03T15:46:51.737Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"7c88827f-2b37-4992-947a-f7c1c2f44071"}
{"level":"error","ts":"2026-11-03T15:46:51.737Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"7c88827f-2b37-4992-947a-f7c1c2f44071","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster not found: certificates.cert-manager.io \"ray-head-cert-ray-cluster\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:46:51.737Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4e06c780-a744-45c0-b88d-e0662a4c8e0a"}
{"level":"info","ts":"2026-11-03T15:46:51.737Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4e06c780-a744-45c0-b88d-e0662a4c8e0a"}
{"level":"info","ts":"2026-11-03T15:46:51.737Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4e06c780-a744-45c0-b88d-e0662a4c8e0a","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:46:51.737Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4e06c780-a744-45c0-b88d-e0662a4c8e0a"}
{"level":"info","ts":"2026-11-03T15:46:51.766Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"namespace":"test-ns-zjpgs","name":"ray-cluster","reconcileID":"062c4941-9241-492d-a9d1-88432e4a82f9","name":"kuberay-gateway-access","namespace":"test-ns-zjpgs","operation":"created"}
{"level":"info","ts":"2026-11-03T15:46:51.786Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4e06c780-a744-45c0-b88d-e0662a4c8e0a","error":"certificate ray-head-cert-ray-cluster not found: certificates.cert-manager.io \"ray-head-cert-ray-cluster\" not found"}
{"level":"error","ts":"2026-11-03T15:46:51.786Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4e06c780-a744-45c0-b88d-e0662a4c8e0a","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster not found: certificates.cert-manager.io \"ray-head-cert-ray-cluster\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:46:51.786Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4e06c780-a744-45c0-b88d-e0662a4c8e0a","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster not found: certificates.cert-manager.io \"ray-head-cert-ray-cluster\" not found"}
{"level":"info","ts":"2026-11-03T15:46:51.786Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4e06c780-a744-45c0-b88d-e0662a4c8e0a","filter labels":["test-ns-zjpgs",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:46:51.787Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4e06c780-a744-45c0-b88d-e0662a4c8e0a","filter labels":["test-ns-zjpgs",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:46:51.787Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4e06c780-a744-45c0-b88d-e0662a4c8e0a","filter labels":["test-ns-zjpgs",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:46:51.787Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4e06c780-a744-45c0-b88d-e0662a4c8e0a","name":"ray-cluster","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{},"conditions":[{"type":"AuthenticationReady","status":"False","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:46:51Z","reason":"AuthenticationFailed","message":"Failed to create authentication resources: autoscaler RoleBinding not found yet"}]},"new status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:46:51Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"False","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:46:51Z","reason":"AuthenticationFailed","message":"Failed to create authentication resources: autoscaler RoleBinding not found yet"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:46:51.820Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"namespace":"test-ns-zjpgs","name":"ray-cluster","reconcileID":"062c4941-9241-492d-a9d1-88432e4a82f9","name":"test-ns-zjpgs-ray-cluster","operation":"created"}
{"level":"info","ts":"2026-11-03T15:46:51.846Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4e06c780-a744-45c0-b88d-e0662a4c8e0a"}
{"level":"error","ts":"2026-11-03T15:46:51.846Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4e06c780-a744-45c0-b88d-e0662a4c8e0a","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster not found: certificates.cert-manager.io \"ray-head-cert-ray-cluster\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:46:51.857Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9bb75530-6895-4ed0-9b33-e7f8eaf76031"}
{"level":"info","ts":"2026-11-03T15:46:51.857Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9bb75530-6895-4ed0-9b33-e7f8eaf76031"}
{"level":"info","ts":"2026-11-03T15:46:51.857Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9bb75530-6895-4ed0-9b33-e7f8eaf76031","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:46:51.857Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9bb75530-6895-4ed0-9b33-e7f8eaf76031"}
{"level":"info","ts":"2026-11-03T15:46:51.872Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"namespace":"test-ns-zjpgs","name":"ray-cluster","reconcileID":"062c4941-9241-492d-a9d1-88432e4a82f9","name":"kube-rbac-proxy-config-ray-cluster","operation":"created"}
{"level":"info","ts":"2026-11-03T15:46:51.906Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9bb75530-6895-4ed0-9b33-e7f8eaf76031","error":"certificate ray-head-cert-ray-cluster not found: certificates.cert-manager.io \"ray-head-cert-ray-cluster\" not found"}
{"level":"error","ts":"2026-11-03T15:46:51.906Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9bb75530-6895-4ed0-9b33-e7f8eaf76031","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster not found: certificates.cert-manager.io \"ray-head-cert-ray-cluster\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:46:51.906Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9bb75530-6895-4ed0-9b33-e7f8eaf76031","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster not found: certificates.cert-manager.io \"ray-head-cert-ray-cluster\" not found"}
{"level":"info","ts":"2026-11-03T15:46:51.906Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9bb75530-6895-4ed0-9b33-e7f8eaf76031","filter labels":["test-ns-zjpgs",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:46:51.906Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9bb75530-6895-4ed0-9b33-e7f8eaf76031","filter labels":["test-ns-zjpgs",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:46:51.906Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9bb75530-6895-4ed0-9b33-e7f8eaf76031","filter labels":["test-ns-zjpgs",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:46:51.906Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9bb75530-6895-4ed0-9b33-e7f8eaf76031"}
{"level":"error","ts":"2026-11-03T15:46:51.906Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9bb75530-6895-4ed0-9b33-e7f8eaf76031","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster not found: certificates.cert-manager.io \"ray-head-cert-ray-cluster\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-03T15:46:51.926Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"namespace":"test-ns-zjpgs","name":"ray-cluster","reconcileID":"062c4941-9241-492d-a9d1-88432e4a82f9","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:167\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:46:51.926Z","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"namespace":"test-ns-zjpgs","name":"ray-cluster","reconcileID":"062c4941-9241-492d-a9d1-88432e4a82f9"}
{"level":"error","ts":"2026-11-03T15:46:51.926Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"namespace":"test-ns-zjpgs","name":"ray-cluster","reconcileID":"062c4941-9241-492d-a9d1-88432e4a82f9","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:46:51.927Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"f2ee333a-7cfb-49f7-b4d8-cf05e3f779f8"}
{"level":"info","ts":"2026-11-03T15:46:51.927Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"f2ee333a-7cfb-49f7-b4d8-cf05e3f779f8"}
{"level":"info","ts":"2026-11-03T15:46:51.927Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"f2ee333a-7cfb-49f7-b4d8-cf05e3f779f8","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:46:51.927Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"f2ee333a-7cfb-49f7-b4d8-cf05e3f779f8"}
{"level":"info","ts":"2026-11-03T15:46:51.932Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"namespace":"test-ns-zjpgs","name":"ray-cluster","reconcileID":"5f9a878e-e824-4600-a0ea-6c6029240279","namespacedName":{"name":"ray-cluster","namespace":"test-ns-zjpgs"}}
{"level":"info","ts":"2026-11-03T15:46:51.932Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"namespace":"test-ns-zjpgs","name":"ray-cluster","reconcileID":"5f9a878e-e824-4600-a0ea-6c6029240279","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:46:51.932Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"namespace":"test-ns-zjpgs","name":"ray-cluster","reconcileID":"5f9a878e-e824-4600-a0ea-6c6029240279","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:46:51.976Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"f2ee333a-7cfb-49f7-b4d8-cf05e3f779f8","error":"certificate ray-head-cert-ray-cluster not found: certificates.cert-manager.io \"ray-head-cert-ray-cluster\" not found"}
{"level":"error","ts":"2026-11-03T15:46:51.976Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"f2ee333a-7cfb-49f7-b4d8-cf05e3f779f8","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster not found: certificates.cert-manager.io \"ray-head-cert-ray-cluster\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:46:51.976Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"f2ee333a-7cfb-49f7-b4d8-cf05e3f779f8","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster not found: certificates.cert-manager.io \"ray-head-cert-ray-cluster\" not found"}
{"level":"info","ts":"2026-11-03T15:46:51.976Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"f2ee333a-7cfb-49f7-b4d8-cf05e3f779f8","filter labels":["test-ns-zjpgs",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:46:51.976Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"f2ee333a-7cfb-49f7-b4d8-cf05e3f779f8","filter labels":["test-ns-zjpgs",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:46:51.976Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"f2ee333a-7cfb-49f7-b4d8-cf05e3f779f8","filter labels":["test-ns-zjpgs",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:46:51.976Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"f2ee333a-7cfb-49f7-b4d8-cf05e3f779f8"}
{"level":"error","ts":"2026-11-03T15:46:51.976Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"f2ee333a-7cfb-49f7-b4d8-cf05e3f779f8","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster not found: certificates.cert-manager.io \"ray-head-cert-ray-cluster\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:46:51.992Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"namespace":"test-ns-zjpgs","name":"ray-cluster","reconcileID":"5f9a878e-e824-4600-a0ea-6c6029240279","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:46:52.017Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"d4478584-a87c-4de8-96bb-973ad0f6c9c9"}
{"level":"info","ts":"2026-11-03T15:46:52.017Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"d4478584-a87c-4de8-96bb-973ad0f6c9c9"}
{"level":"info","ts":"2026-11-03T15:46:52.017Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"d4478584-a87c-4de8-96bb-973ad0f6c9c9","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:46:52.017Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"d4478584-a87c-4de8-96bb-973ad0f6c9c9"}
{"level":"info","ts":"2026-11-03T15:46:52.068Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"d4478584-a87c-4de8-96bb-973ad0f6c9c9","error":"certificate ray-head-cert-ray-cluster is not ready at current generation 1"}
{"level":"error","ts":"2026-11-03T15:46:52.068Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"d4478584-a87c-4de8-96bb-973ad0f6c9c9","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 1","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:46:52.068Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"d4478584-a87c-4de8-96bb-973ad0f6c9c9","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 1"}
{"level":"info","ts":"2026-11-03T15:46:52.068Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"d4478584-a87c-4de8-96bb-973ad0f6c9c9","filter labels":["test-ns-zjpgs",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:46:52.068Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"d4478584-a87c-4de8-96bb-973ad0f6c9c9","filter labels":["test-ns-zjpgs",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:46:52.068Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"d4478584-a87c-4de8-96bb-973ad0f6c9c9","filter labels":["test-ns-zjpgs",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:46:52.068Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"d4478584-a87c-4de8-96bb-973ad0f6c9c9"}
{"level":"error","ts":"2026-11-03T15:46:52.068Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"d4478584-a87c-4de8-96bb-973ad0f6c9c9","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:46:52.149Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9afc9c5c-1e17-46ab-9865-8cd4efcda33f"}
{"level":"info","ts":"2026-11-03T15:46:52.149Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9afc9c5c-1e17-46ab-9865-8cd4efcda33f"}
{"level":"info","ts":"2026-11-03T15:46:52.149Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9afc9c5c-1e17-46ab-9865-8cd4efcda33f","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:46:52.149Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9afc9c5c-1e17-46ab-9865-8cd4efcda33f"}
{"level":"info","ts":"2026-11-03T15:46:52.199Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9afc9c5c-1e17-46ab-9865-8cd4efcda33f","error":"certificate ray-head-cert-ray-cluster is not ready at current generation 1"}
{"level":"error","ts":"2026-11-03T15:46:52.199Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9afc9c5c-1e17-46ab-9865-8cd4efcda33f","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 1","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:46:52.199Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9afc9c5c-1e17-46ab-9865-8cd4efcda33f","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 1"}
{"level":"info","ts":"2026-11-03T15:46:52.199Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9afc9c5c-1e17-46ab-9865-8cd4efcda33f","filter labels":["test-ns-zjpgs",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:46:52.200Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9afc9c5c-1e17-46ab-9865-8cd4efcda33f","filter labels":["test-ns-zjpgs",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:46:52.200Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9afc9c5c-1e17-46ab-9865-8cd4efcda33f","filter labels":["test-ns-zjpgs",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:46:52.200Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9afc9c5c-1e17-46ab-9865-8cd4efcda33f"}
{"level":"error","ts":"2026-11-03T15:46:52.200Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9afc9c5c-1e17-46ab-9865-8cd4efcda33f","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:46:52.235Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"10524022-1126-4bce-8e5c-6054af7b337f","certificate":"ray-head-cert-ray-cluster"}
{"level":"info","ts":"2026-11-03T15:46:52.235Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"10524022-1126-4bce-8e5c-6054af7b337f"}
{"level":"info","ts":"2026-11-03T15:46:52.235Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4229107d-5698-4c8e-bd02-a38d0e4e70cb","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:46:52.361Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"5bb69c32-57ce-47d7-bbc2-d6148be7eed5"}
{"level":"info","ts":"2026-11-03T15:46:52.361Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"5bb69c32-57ce-47d7-bbc2-d6148be7eed5"}
{"level":"info","ts":"2026-11-03T15:46:52.361Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"5bb69c32-57ce-47d7-bbc2-d6148be7eed5","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:46:52.361Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"5bb69c32-57ce-47d7-bbc2-d6148be7eed5"}
{"level":"info","ts":"2026-11-03T15:46:52.411Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"5bb69c32-57ce-47d7-bbc2-d6148be7eed5","error":"certificate ray-head-cert-ray-cluster is not ready at current generation 1"}
{"level":"error","ts":"2026-11-03T15:46:52.411Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"5bb69c32-57ce-47d7-bbc2-d6148be7eed5","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 1","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:46:52.411Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"5bb69c32-57ce-47d7-bbc2-d6148be7eed5","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 1"}
{"level":"info","ts":"2026-11-03T15:46:52.411Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"5bb69c32-57ce-47d7-bbc2-d6148be7eed5","filter labels":["test-ns-zjpgs",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:46:52.411Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"5bb69c32-57ce-47d7-bbc2-d6148be7eed5","filter labels":["test-ns-zjpgs",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:46:52.411Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"5bb69c32-57ce-47d7-bbc2-d6148be7eed5","filter labels":["test-ns-zjpgs",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:46:52.411Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"5bb69c32-57ce-47d7-bbc2-d6148be7eed5"}
{"level":"error","ts":"2026-11-03T15:46:52.411Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"5bb69c32-57ce-47d7-bbc2-d6148be7eed5","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:46:52.686Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4229107d-5698-4c8e-bd02-a38d0e4e70cb","certificate":"ray-head-cert-ray-cluster"}
{"level":"info","ts":"2026-11-03T15:46:52.686Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4229107d-5698-4c8e-bd02-a38d0e4e70cb"}
{"level":"info","ts":"2026-11-03T15:46:52.733Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"3c8bdf99-e8d3-4d3e-889f-d8d6ea27cd0e"}
{"level":"info","ts":"2026-11-03T15:46:52.733Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"3c8bdf99-e8d3-4d3e-889f-d8d6ea27cd0e"}
{"level":"info","ts":"2026-11-03T15:46:52.733Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"3c8bdf99-e8d3-4d3e-889f-d8d6ea27cd0e","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:46:52.733Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"3c8bdf99-e8d3-4d3e-889f-d8d6ea27cd0e"}
{"level":"info","ts":"2026-11-03T15:46:52.783Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"3c8bdf99-e8d3-4d3e-889f-d8d6ea27cd0e","error":"certificate ray-head-cert-ray-cluster is not ready at current generation 1"}
{"level":"error","ts":"2026-11-03T15:46:52.783Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"3c8bdf99-e8d3-4d3e-889f-d8d6ea27cd0e","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 1","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:46:52.783Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"3c8bdf99-e8d3-4d3e-889f-d8d6ea27cd0e","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 1"}
{"level":"info","ts":"2026-11-03T15:46:52.783Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"3c8bdf99-e8d3-4d3e-889f-d8d6ea27cd0e","filter labels":["test-ns-zjpgs",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:46:52.783Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"3c8bdf99-e8d3-4d3e-889f-d8d6ea27cd0e","filter labels":["test-ns-zjpgs",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:46:52.783Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"3c8bdf99-e8d3-4d3e-889f-d8d6ea27cd0e","filter labels":["test-ns-zjpgs",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:46:52.784Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"3c8bdf99-e8d3-4d3e-889f-d8d6ea27cd0e"}
{"level":"error","ts":"2026-11-03T15:46:52.784Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"3c8bdf99-e8d3-4d3e-889f-d8d6ea27cd0e","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:46:53.424Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9ba0ea8b-c288-458d-bb1a-a8fc29277e03"}
{"level":"info","ts":"2026-11-03T15:46:53.424Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9ba0ea8b-c288-458d-bb1a-a8fc29277e03"}
{"level":"info","ts":"2026-11-03T15:46:53.424Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9ba0ea8b-c288-458d-bb1a-a8fc29277e03","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:46:53.424Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9ba0ea8b-c288-458d-bb1a-a8fc29277e03"}
{"level":"info","ts":"2026-11-03T15:46:53.475Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9ba0ea8b-c288-458d-bb1a-a8fc29277e03","error":"certificate ray-head-cert-ray-cluster is not ready at current generation 1"}
{"level":"error","ts":"2026-11-03T15:46:53.475Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9ba0ea8b-c288-458d-bb1a-a8fc29277e03","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 1","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:46:53.475Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9ba0ea8b-c288-458d-bb1a-a8fc29277e03","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 1"}
{"level":"info","ts":"2026-11-03T15:46:53.475Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9ba0ea8b-c288-458d-bb1a-a8fc29277e03","filter labels":["test-ns-zjpgs",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:46:53.475Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9ba0ea8b-c288-458d-bb1a-a8fc29277e03","filter labels":["test-ns-zjpgs",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:46:53.476Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9ba0ea8b-c288-458d-bb1a-a8fc29277e03","filter labels":["test-ns-zjpgs",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:46:53.476Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9ba0ea8b-c288-458d-bb1a-a8fc29277e03"}
{"level":"error","ts":"2026-11-03T15:46:53.476Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9ba0ea8b-c288-458d-bb1a-a8fc29277e03","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:46:54.756Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"6cc33cb2-5181-40bc-b721-6d94e46f8fde"}
{"level":"info","ts":"2026-11-03T15:46:54.756Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"6cc33cb2-5181-40bc-b721-6d94e46f8fde"}
{"level":"info","ts":"2026-11-03T15:46:54.756Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"6cc33cb2-5181-40bc-b721-6d94e46f8fde","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:46:54.756Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"6cc33cb2-5181-40bc-b721-6d94e46f8fde"}
{"level":"info","ts":"2026-11-03T15:46:54.857Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"6cc33cb2-5181-40bc-b721-6d94e46f8fde","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:46:54.857Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"6cc33cb2-5181-40bc-b721-6d94e46f8fde","cluster":"ray-cluster","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-11-03T15:46:54.857Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"6cc33cb2-5181-40bc-b721-6d94e46f8fde","cluster":"ray-cluster","authType":"IntegratedOAuth","serviceAccount":"ray-cluster-oauth-proxy-sa","containerCount":3}
{"level":"info","ts":"2026-11-03T15:46:54.857Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"6cc33cb2-5181-40bc-b721-6d94e46f8fde","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"ray-cluster","ray.io/group":"headgroup","ray.io/identifier":"ray-cluster-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-11-03T15:46:54.857Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"6cc33cb2-5181-40bc-b721-6d94e46f8fde","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080","no-monitor":"true","num-cpus":"0"},"Ray container resource":{"limits":{"cpu":"1","memory":"4G"},"requests":{"cpu":"1","memory":"2G"}}}
{"level":"info","ts":"2026-11-03T15:46:54.857Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"6cc33cb2-5181-40bc-b721-6d94e46f8fde","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --num-cpus=0 "}
{"level":"info","ts":"2026-11-03T15:46:54.857Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"6cc33cb2-5181-40bc-b721-6d94e46f8fde","rayNodeType":"head","generatedCmd":"ulimit -n ${RAY_START_ULIMIT_OPEN_FILES:-65536}; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=4000000000  --metrics-export-port=8080  --no-monitor  --num-cpus=0 "}
{"level":"info","ts":"2026-11-03T15:46:54.857Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"6cc33cb2-5181-40bc-b721-6d94e46f8fde","enabled":true}
{"level":"info","ts":"2026-11-03T15:46:54.977Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"6cc33cb2-5181-40bc-b721-6d94e46f8fde","name":"ray-cluster-head-jhq58"}
{"level":"info","ts":"2026-11-03T15:46:54.978Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"6cc33cb2-5181-40bc-b721-6d94e46f8fde","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-11-03T15:46:54.978Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"6cc33cb2-5181-40bc-b721-6d94e46f8fde","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:46:54.978Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"6cc33cb2-5181-40bc-b721-6d94e46f8fde","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-11-03T15:46:54.978Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"6cc33cb2-5181-40bc-b721-6d94e46f8fde","group":"small-group","indices":{}}
{"level":"info","ts":"2026-11-03T15:46:54.978Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"6cc33cb2-5181-40bc-b721-6d94e46f8fde","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:46:54.978Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"6cc33cb2-5181-40bc-b721-6d94e46f8fde","podName":"ray-cluster-head-jhq58","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:46:54.978Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"6cc33cb2-5181-40bc-b721-6d94e46f8fde","podName":"ray-cluster-head-jhq58","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:46:54.978Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"6cc33cb2-5181-40bc-b721-6d94e46f8fde","name":"ray-cluster","old status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:46:51Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:46:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:46:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ray-cluster-head-jhq58","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:46:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:46:55.084Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"1f62ac4b-e8bd-4233-896f-ef0fe5653d72","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:46:55.084Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"6cf1e47a-5234-428a-ae04-19fa44052625"}
{"level":"info","ts":"2026-11-03T15:46:55.084Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"6cf1e47a-5234-428a-ae04-19fa44052625"}
{"level":"info","ts":"2026-11-03T15:46:55.084Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"6cf1e47a-5234-428a-ae04-19fa44052625","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:46:55.085Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"6cf1e47a-5234-428a-ae04-19fa44052625","Found 1 head Pod":"ray-cluster-head-jhq58","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:46:55.085Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"6cf1e47a-5234-428a-ae04-19fa44052625","head Pod":"ray-cluster-head-jhq58","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-jhq58. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:46:55.085Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"6cf1e47a-5234-428a-ae04-19fa44052625","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-11-03T15:46:55.085Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"6cf1e47a-5234-428a-ae04-19fa44052625","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:46:55.085Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"6cf1e47a-5234-428a-ae04-19fa44052625","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-11-03T15:46:55.085Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"6cf1e47a-5234-428a-ae04-19fa44052625","group":"small-group","indices":{}}
{"level":"info","ts":"2026-11-03T15:46:55.085Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"6cf1e47a-5234-428a-ae04-19fa44052625","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:46:55.085Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"6cf1e47a-5234-428a-ae04-19fa44052625","podName":"ray-cluster-head-jhq58","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:46:55.085Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"6cf1e47a-5234-428a-ae04-19fa44052625","podName":"ray-cluster-head-jhq58","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:46:55.085Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"6cf1e47a-5234-428a-ae04-19fa44052625","name":"ray-cluster","old status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:46:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ray-cluster-head-jhq58","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:46:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"Unknown","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:46:55Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ray-cluster-head-jhq58","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:46:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"PodInitializing","message":"containers with unready status: [ray-head autoscaler kube-rbac-proxy]; autoscaler: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:46:55.693Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"27f981ca-69fe-418c-95f8-fd41994ab052"}
{"level":"info","ts":"2026-11-03T15:46:55.693Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"27f981ca-69fe-418c-95f8-fd41994ab052"}
{"level":"info","ts":"2026-11-03T15:46:55.693Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"27f981ca-69fe-418c-95f8-fd41994ab052","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:46:55.693Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"27f981ca-69fe-418c-95f8-fd41994ab052","Found 1 head Pod":"ray-cluster-head-jhq58","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:46:55.693Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"27f981ca-69fe-418c-95f8-fd41994ab052","head Pod":"ray-cluster-head-jhq58","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-jhq58. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:46:55.693Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"27f981ca-69fe-418c-95f8-fd41994ab052","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-11-03T15:46:55.693Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"27f981ca-69fe-418c-95f8-fd41994ab052","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:46:55.693Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"27f981ca-69fe-418c-95f8-fd41994ab052","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-11-03T15:46:55.693Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"27f981ca-69fe-418c-95f8-fd41994ab052","group":"small-group","indices":{}}
{"level":"info","ts":"2026-11-03T15:46:55.693Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"27f981ca-69fe-418c-95f8-fd41994ab052","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:46:55.693Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"27f981ca-69fe-418c-95f8-fd41994ab052","podName":"ray-cluster-head-jhq58","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:46:55.693Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"27f981ca-69fe-418c-95f8-fd41994ab052","podName":"ray-cluster-head-jhq58","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:46:55.693Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"27f981ca-69fe-418c-95f8-fd41994ab052","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:46:55.693Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"27f981ca-69fe-418c-95f8-fd41994ab052","seconds":300}
{"level":"info","ts":"2026-11-03T15:46:55.909Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"514dd5a7-100c-4082-a6a3-9a93a90ae8ad","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:46:56.127Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"c738532a-2cf8-465f-ba95-a2975bd451b8"}
{"level":"info","ts":"2026-11-03T15:46:56.127Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"c738532a-2cf8-465f-ba95-a2975bd451b8"}
{"level":"info","ts":"2026-11-03T15:46:56.127Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"c738532a-2cf8-465f-ba95-a2975bd451b8","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:46:56.127Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"c738532a-2cf8-465f-ba95-a2975bd451b8","Found 1 head Pod":"ray-cluster-head-9lzd6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:46:56.127Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"c738532a-2cf8-465f-ba95-a2975bd451b8","head Pod":"ray-cluster-head-9lzd6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-9lzd6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:46:56.127Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"c738532a-2cf8-465f-ba95-a2975bd451b8","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-11-03T15:46:56.127Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"c738532a-2cf8-465f-ba95-a2975bd451b8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:46:56.127Z","logger":"controllers.RayCluster","msg":"Deleting pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"c738532a-2cf8-465f-ba95-a2975bd451b8","namespace":"test-ns-4jsqd","name":"ray-cluster-small-group-worker-vwxmp"}
{"level":"info","ts":"2026-11-03T15:46:56.177Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"c738532a-2cf8-465f-ba95-a2975bd451b8","The worker Pod has already been deleted":"ray-cluster-small-group-worker-vwxmp"}
{"level":"info","ts":"2026-11-03T15:46:56.177Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"c738532a-2cf8-465f-ba95-a2975bd451b8","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-11-03T15:46:56.177Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"c738532a-2cf8-465f-ba95-a2975bd451b8","group":"small-group","indices":{}}
{"level":"info","ts":"2026-11-03T15:46:56.177Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"c738532a-2cf8-465f-ba95-a2975bd451b8","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:46:56.177Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"c738532a-2cf8-465f-ba95-a2975bd451b8","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:46:56.177Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"c738532a-2cf8-465f-ba95-a2975bd451b8","seconds":300}
{"level":"info","ts":"2026-11-03T15:46:56.305Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"0f3a0278-acc7-4045-8ec5-04293958c518"}
{"level":"info","ts":"2026-11-03T15:46:56.305Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"0f3a0278-acc7-4045-8ec5-04293958c518"}
{"level":"info","ts":"2026-11-03T15:46:56.305Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"0f3a0278-acc7-4045-8ec5-04293958c518","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:46:56.305Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"0f3a0278-acc7-4045-8ec5-04293958c518","Found 1 head Pod":"ray-cluster-head-9lzd6","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:46:56.305Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"0f3a0278-acc7-4045-8ec5-04293958c518","head Pod":"ray-cluster-head-9lzd6","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-9lzd6. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:46:56.305Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"0f3a0278-acc7-4045-8ec5-04293958c518","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-11-03T15:46:56.305Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"0f3a0278-acc7-4045-8ec5-04293958c518","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:46:56.305Z","logger":"controllers.RayCluster","msg":"Deleting pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"0f3a0278-acc7-4045-8ec5-04293958c518","namespace":"test-ns-4jsqd","name":"ray-cluster-small-group-worker-vwxmp"}
{"level":"info","ts":"2026-11-03T15:46:56.356Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"0f3a0278-acc7-4045-8ec5-04293958c518","The worker Pod has already been deleted":"ray-cluster-small-group-worker-vwxmp"}
{"level":"info","ts":"2026-11-03T15:46:56.356Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"0f3a0278-acc7-4045-8ec5-04293958c518","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-11-03T15:46:56.356Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"0f3a0278-acc7-4045-8ec5-04293958c518","group":"small-group","indices":{}}
{"level":"info","ts":"2026-11-03T15:46:56.356Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"0f3a0278-acc7-4045-8ec5-04293958c518","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:46:56.357Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"0f3a0278-acc7-4045-8ec5-04293958c518","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:46:56.357Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"0f3a0278-acc7-4045-8ec5-04293958c518","seconds":300}
{"level":"info","ts":"2026-11-03T15:46:56.833Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"49a0c4d3-8111-4a7a-9b6c-2fb98559ec1d"}
{"level":"info","ts":"2026-11-03T15:46:56.833Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"49a0c4d3-8111-4a7a-9b6c-2fb98559ec1d"}
{"level":"info","ts":"2026-11-03T15:46:56.833Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"49a0c4d3-8111-4a7a-9b6c-2fb98559ec1d","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:46:56.833Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"49a0c4d3-8111-4a7a-9b6c-2fb98559ec1d","Found 1 head Pod":"ray-cluster-head-jhq58","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:46:56.833Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"49a0c4d3-8111-4a7a-9b6c-2fb98559ec1d","head Pod":"ray-cluster-head-jhq58","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-jhq58. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:46:56.833Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"49a0c4d3-8111-4a7a-9b6c-2fb98559ec1d","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-11-03T15:46:56.833Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"49a0c4d3-8111-4a7a-9b6c-2fb98559ec1d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:46:56.833Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"49a0c4d3-8111-4a7a-9b6c-2fb98559ec1d","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-11-03T15:46:56.833Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"49a0c4d3-8111-4a7a-9b6c-2fb98559ec1d","group":"small-group","indices":{}}
{"level":"info","ts":"2026-11-03T15:46:56.833Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"49a0c4d3-8111-4a7a-9b6c-2fb98559ec1d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:46:56.834Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"49a0c4d3-8111-4a7a-9b6c-2fb98559ec1d","podName":"ray-cluster-head-jhq58","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:46:56.834Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"49a0c4d3-8111-4a7a-9b6c-2fb98559ec1d","podName":"ray-cluster-head-jhq58","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:46:56.834Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"49a0c4d3-8111-4a7a-9b6c-2fb98559ec1d","name":"ray-cluster","old status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:46:55Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"ray-cluster-head-jhq58","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:46:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"PodInitializing","message":"containers with unready status: [ray-head autoscaler kube-rbac-proxy]; autoscaler: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:46:56Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"ray-cluster-head-jhq58","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:46:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"PodInitializing","message":"containers with unready status: [ray-head autoscaler kube-rbac-proxy]; autoscaler: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:46:56.895Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"18a6797a-d512-434b-a7c1-01f6dccce495","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:46:57.199Z","logger":"controllers.RayClusterMTLS","msg":"Updating head certificate","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"18a6797a-d512-434b-a7c1-01f6dccce495","certificate":"ray-head-cert-ray-cluster","ipAddresses":["10.132.0.54","127.0.0.1"]}
{"level":"info","ts":"2026-11-03T15:46:57.612Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"18a6797a-d512-434b-a7c1-01f6dccce495","certificate":"ray-head-cert-ray-cluster"}
{"level":"info","ts":"2026-11-03T15:46:57.612Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"18a6797a-d512-434b-a7c1-01f6dccce495"}
{"level":"info","ts":"2026-11-03T15:46:58.895Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"bdefcdab-23c8-4130-bf7e-dd5af114ee8e"}
{"level":"info","ts":"2026-11-03T15:46:58.895Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"bdefcdab-23c8-4130-bf7e-dd5af114ee8e"}
{"level":"info","ts":"2026-11-03T15:46:58.895Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"bdefcdab-23c8-4130-bf7e-dd5af114ee8e","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:46:58.895Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"bdefcdab-23c8-4130-bf7e-dd5af114ee8e","Found 1 head Pod":"ray-cluster-head-jhq58","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:46:58.896Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"bdefcdab-23c8-4130-bf7e-dd5af114ee8e","head Pod":"ray-cluster-head-jhq58","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-jhq58. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:46:58.896Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"bdefcdab-23c8-4130-bf7e-dd5af114ee8e","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-11-03T15:46:58.896Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"bdefcdab-23c8-4130-bf7e-dd5af114ee8e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:46:58.896Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"bdefcdab-23c8-4130-bf7e-dd5af114ee8e","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-11-03T15:46:58.896Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"bdefcdab-23c8-4130-bf7e-dd5af114ee8e","group":"small-group","indices":{}}
{"level":"info","ts":"2026-11-03T15:46:58.896Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"bdefcdab-23c8-4130-bf7e-dd5af114ee8e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:46:58.896Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"bdefcdab-23c8-4130-bf7e-dd5af114ee8e","podName":"ray-cluster-head-jhq58","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:46:58.896Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"bdefcdab-23c8-4130-bf7e-dd5af114ee8e","podName":"ray-cluster-head-jhq58","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:46:58.896Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"bdefcdab-23c8-4130-bf7e-dd5af114ee8e","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:46:58.896Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"bdefcdab-23c8-4130-bf7e-dd5af114ee8e","seconds":300}
{"level":"info","ts":"2026-11-03T15:47:21.403Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"8111943e-d72c-4a31-8925-c1d5702eab10","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:47:21.710Z","logger":"controllers.RayClusterMTLS","msg":"Updating head certificate","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"8111943e-d72c-4a31-8925-c1d5702eab10","certificate":"ray-head-cert-ray-cluster","ipAddresses":["127.0.0.1"]}
{"level":"error","ts":"2026-11-03T15:47:21.759Z","logger":"events","msg":"Server rejected event (will not retry!)","event":{"namespace":"test-ns-4jsqd","name":"ray-cluster.18e4939a1c586745"},"error":"events.events.k8s.io \"ray-cluster.18e4939a1c586745\" is forbidden: unable to create new content in namespace test-ns-4jsqd because it is being terminated","stacktrace":"k8s.io/client-go/tools/events.recordEvent\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:270\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).attemptRecording\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:211\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).recordToSink.func1\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:200"}
{"level":"info","ts":"2026-11-03T15:47:22.125Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"8111943e-d72c-4a31-8925-c1d5702eab10","certificate":"ray-head-cert-ray-cluster"}
{"level":"info","ts":"2026-11-03T15:47:22.125Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"8111943e-d72c-4a31-8925-c1d5702eab10"}
{"level":"error","ts":"2026-11-03T15:47:22.224Z","logger":"events","msg":"Server rejected event (will not retry!)","event":{"namespace":"test-ns-4jsqd","name":"ray-cluster.18e493930de3d390"},"error":"events.events.k8s.io \"ray-cluster.18e493930de3d390\" is forbidden: unable to create new content in namespace test-ns-4jsqd because it is being terminated","stacktrace":"k8s.io/client-go/tools/events.recordEvent\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:270\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).attemptRecording\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:211\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).recordToSink.func1\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:200"}
{"level":"info","ts":"2026-11-03T15:47:27.046Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"c325918b-226e-4421-892e-fcbb60a8309b"}
{"level":"info","ts":"2026-11-03T15:47:27.046Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"c325918b-226e-4421-892e-fcbb60a8309b"}
{"level":"info","ts":"2026-11-03T15:47:27.046Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"c325918b-226e-4421-892e-fcbb60a8309b","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:47:27.046Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"c325918b-226e-4421-892e-fcbb60a8309b","Found 1 head Pod":"ray-cluster-head-9lzd6","Pod status":"Failed","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"&ContainerStateTerminated{ExitCode:1,Signal:0,Reason:Error,Message:,StartedAt:2026-11-03 15:42:27 +0000 UTC,FinishedAt:2026-11-03 15:46:59 +0000 UTC,ContainerID:cri-o://3d71ec593d30549d0538c61a5c09d5d3672e5e9f032c48bb229dd4a59babe88d,}"}
{"level":"info","ts":"2026-11-03T15:47:27.046Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"c325918b-226e-4421-892e-fcbb60a8309b","head Pod":"ray-cluster-head-9lzd6","shouldDelete":true,"reason":"The head Pod ray-cluster-head-9lzd6 status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary."}
{"level":"error","ts":"2026-11-03T15:47:27.097Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"c325918b-226e-4421-892e-fcbb60a8309b","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"The head Pod ray-cluster-head-9lzd6 status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary.","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:47:27.097Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"c325918b-226e-4421-892e-fcbb60a8309b","error":"The head Pod ray-cluster-head-9lzd6 status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary."}
{"level":"info","ts":"2026-11-03T15:47:27.097Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"c325918b-226e-4421-892e-fcbb60a8309b","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:46:50Z","stateTransitionTimes":{"ready":"2026-11-03T15:42:45Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.56","serviceIP":"10.134.0.56","podName":"ray-cluster-head-9lzd6","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-11-03T15:40:56Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:42:45Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:42:45Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":5},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:47:27Z","stateTransitionTimes":{"ready":"2026-11-03T15:42:45Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.56","serviceIP":"10.134.0.56","podName":"ray-cluster-head-9lzd6","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-11-03T15:40:56Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:47:27Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:42:45Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":5}}
{"level":"error","ts":"2026-11-03T15:47:27.146Z","logger":"events","msg":"Server rejected event (will not retry!)","event":{"namespace":"test-ns-4jsqd","name":"ray-cluster.18e4939b5d760cac"},"error":"events.events.k8s.io \"ray-cluster.18e4939b5d760cac\" is forbidden: unable to create new content in namespace test-ns-4jsqd because it is being terminated","stacktrace":"k8s.io/client-go/tools/events.recordEvent\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:270\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).attemptRecording\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:211\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).recordToSink.func1\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:200"}
{"level":"info","ts":"2026-11-03T15:47:27.162Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"c325918b-226e-4421-892e-fcbb60a8309b"}
{"level":"error","ts":"2026-11-03T15:47:27.162Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"c325918b-226e-4421-892e-fcbb60a8309b","error":"The head Pod ray-cluster-head-9lzd6 status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary.","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:47:27.163Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"a0721866-0edd-402e-97a7-9a95e1768cf1","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:47:27.168Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b5901c1b-2229-49d9-823a-fa2e09af346c"}
{"level":"info","ts":"2026-11-03T15:47:27.168Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b5901c1b-2229-49d9-823a-fa2e09af346c"}
{"level":"info","ts":"2026-11-03T15:47:27.168Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b5901c1b-2229-49d9-823a-fa2e09af346c","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:47:27.168Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b5901c1b-2229-49d9-823a-fa2e09af346c","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-11-03T15:47:27.168Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b5901c1b-2229-49d9-823a-fa2e09af346c","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-11-03T15:47:27.168Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b5901c1b-2229-49d9-823a-fa2e09af346c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:47:27.168Z","logger":"controllers.RayCluster","msg":"Deleting pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b5901c1b-2229-49d9-823a-fa2e09af346c","namespace":"test-ns-4jsqd","name":"ray-cluster-small-group-worker-vwxmp"}
{"level":"info","ts":"2026-11-03T15:47:27.218Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b5901c1b-2229-49d9-823a-fa2e09af346c","The worker Pod has already been deleted":"ray-cluster-small-group-worker-vwxmp"}
{"level":"info","ts":"2026-11-03T15:47:27.218Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b5901c1b-2229-49d9-823a-fa2e09af346c","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-11-03T15:47:27.218Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b5901c1b-2229-49d9-823a-fa2e09af346c","group":"small-group","indices":{}}
{"level":"info","ts":"2026-11-03T15:47:27.218Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b5901c1b-2229-49d9-823a-fa2e09af346c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:47:27.218Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b5901c1b-2229-49d9-823a-fa2e09af346c","podName":"ray-cluster-head-9lzd6","pod Status.Phase":"Failed"}
{"level":"info","ts":"2026-11-03T15:47:27.218Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b5901c1b-2229-49d9-823a-fa2e09af346c","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:47:27.218Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b5901c1b-2229-49d9-823a-fa2e09af346c","seconds":300}
{"level":"info","ts":"2026-11-03T15:47:27.256Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"7fbe34fb-c8e3-4067-b5bb-4bf961073f6e"}
{"level":"info","ts":"2026-11-03T15:47:27.256Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"7fbe34fb-c8e3-4067-b5bb-4bf961073f6e"}
{"level":"info","ts":"2026-11-03T15:47:27.256Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"7fbe34fb-c8e3-4067-b5bb-4bf961073f6e","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:47:27.256Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"7fbe34fb-c8e3-4067-b5bb-4bf961073f6e","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-11-03T15:47:27.256Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"7fbe34fb-c8e3-4067-b5bb-4bf961073f6e","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-11-03T15:47:27.256Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"7fbe34fb-c8e3-4067-b5bb-4bf961073f6e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:47:27.256Z","logger":"controllers.RayCluster","msg":"Deleting pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"7fbe34fb-c8e3-4067-b5bb-4bf961073f6e","namespace":"test-ns-4jsqd","name":"ray-cluster-small-group-worker-vwxmp"}
{"level":"info","ts":"2026-11-03T15:47:27.306Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"7fbe34fb-c8e3-4067-b5bb-4bf961073f6e","The worker Pod has already been deleted":"ray-cluster-small-group-worker-vwxmp"}
{"level":"info","ts":"2026-11-03T15:47:27.306Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"7fbe34fb-c8e3-4067-b5bb-4bf961073f6e","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-11-03T15:47:27.306Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"7fbe34fb-c8e3-4067-b5bb-4bf961073f6e","group":"small-group","indices":{}}
{"level":"info","ts":"2026-11-03T15:47:27.306Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"7fbe34fb-c8e3-4067-b5bb-4bf961073f6e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:47:27.306Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"7fbe34fb-c8e3-4067-b5bb-4bf961073f6e","filter labels":["test-ns-4jsqd",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:47:27.306Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"7fbe34fb-c8e3-4067-b5bb-4bf961073f6e","filter labels":["test-ns-4jsqd",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:47:27.306Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"7fbe34fb-c8e3-4067-b5bb-4bf961073f6e","filter labels":["test-ns-4jsqd",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:47:27.306Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"7fbe34fb-c8e3-4067-b5bb-4bf961073f6e","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:47:27Z","stateTransitionTimes":{"ready":"2026-11-03T15:42:45Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.56","serviceIP":"10.134.0.56","podName":"ray-cluster-head-9lzd6","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-11-03T15:40:56Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:47:27Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:42:45Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":5},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:47:27Z","stateTransitionTimes":{"ready":"2026-11-03T15:42:45Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-11-03T15:40:56Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:47:27Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:42:45Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:40:56Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":5}}
{"level":"info","ts":"2026-11-03T15:47:27.367Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b25b8c40-14d9-412c-bd02-054f080afdfd"}
{"level":"info","ts":"2026-11-03T15:47:27.367Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b25b8c40-14d9-412c-bd02-054f080afdfd"}
{"level":"info","ts":"2026-11-03T15:47:27.367Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b25b8c40-14d9-412c-bd02-054f080afdfd","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:47:27.367Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b25b8c40-14d9-412c-bd02-054f080afdfd"}
{"level":"info","ts":"2026-11-03T15:47:27.417Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b25b8c40-14d9-412c-bd02-054f080afdfd","error":"certificate ray-head-cert-ray-cluster is not ready at current generation 3"}
{"level":"error","ts":"2026-11-03T15:47:27.417Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b25b8c40-14d9-412c-bd02-054f080afdfd","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 3","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:47:27.417Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b25b8c40-14d9-412c-bd02-054f080afdfd","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 3"}
{"level":"info","ts":"2026-11-03T15:47:27.418Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b25b8c40-14d9-412c-bd02-054f080afdfd","filter labels":["test-ns-4jsqd",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:47:27.418Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b25b8c40-14d9-412c-bd02-054f080afdfd","filter labels":["test-ns-4jsqd",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:47:27.418Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b25b8c40-14d9-412c-bd02-054f080afdfd","filter labels":["test-ns-4jsqd",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:47:27.418Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b25b8c40-14d9-412c-bd02-054f080afdfd"}
{"level":"error","ts":"2026-11-03T15:47:27.418Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b25b8c40-14d9-412c-bd02-054f080afdfd","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 3","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:47:27.423Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"d0a00855-15b2-4d57-8449-8c6f1301bc32"}
{"level":"info","ts":"2026-11-03T15:47:27.423Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"d0a00855-15b2-4d57-8449-8c6f1301bc32"}
{"level":"info","ts":"2026-11-03T15:47:27.423Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"d0a00855-15b2-4d57-8449-8c6f1301bc32","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:47:27.423Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"d0a00855-15b2-4d57-8449-8c6f1301bc32"}
{"level":"info","ts":"2026-11-03T15:47:27.473Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"d0a00855-15b2-4d57-8449-8c6f1301bc32","error":"certificate ray-head-cert-ray-cluster is not ready at current generation 3"}
{"level":"error","ts":"2026-11-03T15:47:27.473Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"d0a00855-15b2-4d57-8449-8c6f1301bc32","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 3","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:47:27.473Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"d0a00855-15b2-4d57-8449-8c6f1301bc32","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 3"}
{"level":"info","ts":"2026-11-03T15:47:27.474Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"d0a00855-15b2-4d57-8449-8c6f1301bc32","filter labels":["test-ns-4jsqd",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:47:27.474Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"d0a00855-15b2-4d57-8449-8c6f1301bc32","filter labels":["test-ns-4jsqd",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:47:27.474Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"d0a00855-15b2-4d57-8449-8c6f1301bc32","filter labels":["test-ns-4jsqd",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:47:27.474Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"d0a00855-15b2-4d57-8449-8c6f1301bc32"}
{"level":"error","ts":"2026-11-03T15:47:27.474Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"d0a00855-15b2-4d57-8449-8c6f1301bc32","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 3","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:47:27.484Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"dc14cb5c-a454-4fa2-9c59-192b5edbe5bd"}
{"level":"info","ts":"2026-11-03T15:47:27.484Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"dc14cb5c-a454-4fa2-9c59-192b5edbe5bd"}
{"level":"info","ts":"2026-11-03T15:47:27.484Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"dc14cb5c-a454-4fa2-9c59-192b5edbe5bd","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:47:27.484Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"dc14cb5c-a454-4fa2-9c59-192b5edbe5bd"}
{"level":"info","ts":"2026-11-03T15:47:27.535Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"dc14cb5c-a454-4fa2-9c59-192b5edbe5bd","error":"certificate ray-head-cert-ray-cluster is not ready at current generation 3"}
{"level":"error","ts":"2026-11-03T15:47:27.535Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"dc14cb5c-a454-4fa2-9c59-192b5edbe5bd","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 3","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:47:27.535Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"dc14cb5c-a454-4fa2-9c59-192b5edbe5bd","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 3"}
{"level":"info","ts":"2026-11-03T15:47:27.535Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"dc14cb5c-a454-4fa2-9c59-192b5edbe5bd","filter labels":["test-ns-4jsqd",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:47:27.535Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"dc14cb5c-a454-4fa2-9c59-192b5edbe5bd","filter labels":["test-ns-4jsqd",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:47:27.535Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"dc14cb5c-a454-4fa2-9c59-192b5edbe5bd","filter labels":["test-ns-4jsqd",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:47:27.535Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"dc14cb5c-a454-4fa2-9c59-192b5edbe5bd"}
{"level":"error","ts":"2026-11-03T15:47:27.535Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"dc14cb5c-a454-4fa2-9c59-192b5edbe5bd","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 3","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:47:27.555Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"90aca63c-55be-48d2-ba2a-12a81e785370"}
{"level":"info","ts":"2026-11-03T15:47:27.555Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"90aca63c-55be-48d2-ba2a-12a81e785370"}
{"level":"info","ts":"2026-11-03T15:47:27.555Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"90aca63c-55be-48d2-ba2a-12a81e785370","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:47:27.556Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"90aca63c-55be-48d2-ba2a-12a81e785370"}
{"level":"info","ts":"2026-11-03T15:47:27.606Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"90aca63c-55be-48d2-ba2a-12a81e785370","error":"certificate ray-head-cert-ray-cluster is not ready at current generation 3"}
{"level":"error","ts":"2026-11-03T15:47:27.606Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"90aca63c-55be-48d2-ba2a-12a81e785370","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 3","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:47:27.606Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"90aca63c-55be-48d2-ba2a-12a81e785370","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 3"}
{"level":"info","ts":"2026-11-03T15:47:27.606Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"90aca63c-55be-48d2-ba2a-12a81e785370","filter labels":["test-ns-4jsqd",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:47:27.606Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"90aca63c-55be-48d2-ba2a-12a81e785370","filter labels":["test-ns-4jsqd",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:47:27.606Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"90aca63c-55be-48d2-ba2a-12a81e785370","filter labels":["test-ns-4jsqd",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:47:27.607Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"90aca63c-55be-48d2-ba2a-12a81e785370"}
{"level":"error","ts":"2026-11-03T15:47:27.607Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"90aca63c-55be-48d2-ba2a-12a81e785370","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 3","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:47:27.647Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"27eef96c-3999-47a6-8f6d-cb62f14c6a9e"}
{"level":"info","ts":"2026-11-03T15:47:27.647Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"27eef96c-3999-47a6-8f6d-cb62f14c6a9e"}
{"level":"info","ts":"2026-11-03T15:47:27.647Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"27eef96c-3999-47a6-8f6d-cb62f14c6a9e","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:47:27.647Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"27eef96c-3999-47a6-8f6d-cb62f14c6a9e"}
{"level":"info","ts":"2026-11-03T15:47:27.698Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"27eef96c-3999-47a6-8f6d-cb62f14c6a9e","error":"certificate ray-head-cert-ray-cluster is not ready at current generation 3"}
{"level":"error","ts":"2026-11-03T15:47:27.698Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"27eef96c-3999-47a6-8f6d-cb62f14c6a9e","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 3","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:47:27.698Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"27eef96c-3999-47a6-8f6d-cb62f14c6a9e","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 3"}
{"level":"info","ts":"2026-11-03T15:47:27.698Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"27eef96c-3999-47a6-8f6d-cb62f14c6a9e","filter labels":["test-ns-4jsqd",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:47:27.698Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"27eef96c-3999-47a6-8f6d-cb62f14c6a9e","filter labels":["test-ns-4jsqd",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:47:27.698Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"27eef96c-3999-47a6-8f6d-cb62f14c6a9e","filter labels":["test-ns-4jsqd",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:47:27.698Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"27eef96c-3999-47a6-8f6d-cb62f14c6a9e"}
{"level":"error","ts":"2026-11-03T15:47:27.698Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"27eef96c-3999-47a6-8f6d-cb62f14c6a9e","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 3","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:47:27.772Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"a0721866-0edd-402e-97a7-9a95e1768cf1","certificate":"ray-head-cert-ray-cluster"}
{"level":"info","ts":"2026-11-03T15:47:27.772Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"a0721866-0edd-402e-97a7-9a95e1768cf1"}
{"level":"info","ts":"2026-11-03T15:47:27.772Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"d3db1b4f-d4b6-4a63-b8e0-7edd0599771c","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:47:27.778Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"8cb0d32e-c191-40fe-a6b1-32ea30578a0e"}
{"level":"info","ts":"2026-11-03T15:47:27.778Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"8cb0d32e-c191-40fe-a6b1-32ea30578a0e"}
{"level":"info","ts":"2026-11-03T15:47:27.778Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"8cb0d32e-c191-40fe-a6b1-32ea30578a0e","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:47:27.778Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"8cb0d32e-c191-40fe-a6b1-32ea30578a0e"}
{"level":"error","ts":"2026-11-03T15:47:27.822Z","logger":"events","msg":"Server rejected event (will not retry!)","event":{"namespace":"test-ns-4jsqd","name":"ray-cluster.18e4939b85b6f6d4"},"error":"events.events.k8s.io \"ray-cluster.18e4939b85b6f6d4\" is forbidden: unable to create new content in namespace test-ns-4jsqd because it is being terminated","stacktrace":"k8s.io/client-go/tools/events.recordEvent\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:270\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).attemptRecording\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:211\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).recordToSink.func1\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:200"}
{"level":"info","ts":"2026-11-03T15:47:27.829Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"8cb0d32e-c191-40fe-a6b1-32ea30578a0e","error":"certificate ray-head-cert-ray-cluster is not ready at current generation 3"}
{"level":"error","ts":"2026-11-03T15:47:27.829Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"8cb0d32e-c191-40fe-a6b1-32ea30578a0e","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 3","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:47:27.829Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"8cb0d32e-c191-40fe-a6b1-32ea30578a0e","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 3"}
{"level":"info","ts":"2026-11-03T15:47:27.829Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"8cb0d32e-c191-40fe-a6b1-32ea30578a0e","filter labels":["test-ns-4jsqd",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:47:27.829Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"8cb0d32e-c191-40fe-a6b1-32ea30578a0e","filter labels":["test-ns-4jsqd",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:47:27.829Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"8cb0d32e-c191-40fe-a6b1-32ea30578a0e","filter labels":["test-ns-4jsqd",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:47:27.829Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"8cb0d32e-c191-40fe-a6b1-32ea30578a0e"}
{"level":"error","ts":"2026-11-03T15:47:27.829Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"8cb0d32e-c191-40fe-a6b1-32ea30578a0e","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 3","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:47:27.990Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b37465c8-605f-43ff-9b9e-f162fb296e49"}
{"level":"info","ts":"2026-11-03T15:47:27.990Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b37465c8-605f-43ff-9b9e-f162fb296e49"}
{"level":"info","ts":"2026-11-03T15:47:27.990Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b37465c8-605f-43ff-9b9e-f162fb296e49","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:47:27.990Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b37465c8-605f-43ff-9b9e-f162fb296e49"}
{"level":"info","ts":"2026-11-03T15:47:28.046Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b37465c8-605f-43ff-9b9e-f162fb296e49","error":"certificate ray-head-cert-ray-cluster is not ready at current generation 3"}
{"level":"error","ts":"2026-11-03T15:47:28.046Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b37465c8-605f-43ff-9b9e-f162fb296e49","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 3","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:47:28.046Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b37465c8-605f-43ff-9b9e-f162fb296e49","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 3"}
{"level":"info","ts":"2026-11-03T15:47:28.046Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b37465c8-605f-43ff-9b9e-f162fb296e49","filter labels":["test-ns-4jsqd",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:47:28.046Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b37465c8-605f-43ff-9b9e-f162fb296e49","filter labels":["test-ns-4jsqd",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:47:28.046Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b37465c8-605f-43ff-9b9e-f162fb296e49","filter labels":["test-ns-4jsqd",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:47:28.046Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b37465c8-605f-43ff-9b9e-f162fb296e49"}
{"level":"error","ts":"2026-11-03T15:47:28.046Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b37465c8-605f-43ff-9b9e-f162fb296e49","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 3","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:47:28.367Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"206c7314-b0a0-4a16-92ec-0299bbff5934"}
{"level":"info","ts":"2026-11-03T15:47:28.367Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"206c7314-b0a0-4a16-92ec-0299bbff5934"}
{"level":"info","ts":"2026-11-03T15:47:28.367Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"206c7314-b0a0-4a16-92ec-0299bbff5934","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:47:28.367Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"206c7314-b0a0-4a16-92ec-0299bbff5934"}
{"level":"info","ts":"2026-11-03T15:47:28.381Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"d3db1b4f-d4b6-4a63-b8e0-7edd0599771c","certificate":"ray-head-cert-ray-cluster"}
{"level":"info","ts":"2026-11-03T15:47:28.381Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"d3db1b4f-d4b6-4a63-b8e0-7edd0599771c"}
{"level":"info","ts":"2026-11-03T15:47:28.381Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"db64301e-f449-4495-8b5b-daa2ea809aab","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:47:28.418Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"206c7314-b0a0-4a16-92ec-0299bbff5934","error":"certificate ray-head-cert-ray-cluster is not ready at current generation 3"}
{"level":"error","ts":"2026-11-03T15:47:28.418Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"206c7314-b0a0-4a16-92ec-0299bbff5934","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 3","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:47:28.418Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"206c7314-b0a0-4a16-92ec-0299bbff5934","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 3"}
{"level":"info","ts":"2026-11-03T15:47:28.418Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"206c7314-b0a0-4a16-92ec-0299bbff5934","filter labels":["test-ns-4jsqd",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:47:28.418Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"206c7314-b0a0-4a16-92ec-0299bbff5934","filter labels":["test-ns-4jsqd",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:47:28.418Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"206c7314-b0a0-4a16-92ec-0299bbff5934","filter labels":["test-ns-4jsqd",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:47:28.418Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"206c7314-b0a0-4a16-92ec-0299bbff5934"}
{"level":"error","ts":"2026-11-03T15:47:28.418Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"206c7314-b0a0-4a16-92ec-0299bbff5934","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 3","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-03T15:47:28.430Z","logger":"events","msg":"Server rejected event (will not retry!)","event":{"namespace":"test-ns-4jsqd","name":"ray-cluster.18e4939ba9ff7fd0"},"error":"events.events.k8s.io \"ray-cluster.18e4939ba9ff7fd0\" is forbidden: unable to create new content in namespace test-ns-4jsqd because it is being terminated","stacktrace":"k8s.io/client-go/tools/events.recordEvent\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:270\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).attemptRecording\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:211\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).recordToSink.func1\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:200"}
{"level":"info","ts":"2026-11-03T15:47:29.059Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"52ebcea5-c4b5-4d3b-8987-e706e80728e3"}
{"level":"info","ts":"2026-11-03T15:47:29.059Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"52ebcea5-c4b5-4d3b-8987-e706e80728e3"}
{"level":"info","ts":"2026-11-03T15:47:29.059Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"52ebcea5-c4b5-4d3b-8987-e706e80728e3","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:47:29.059Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"52ebcea5-c4b5-4d3b-8987-e706e80728e3"}
{"level":"info","ts":"2026-11-03T15:47:29.110Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"52ebcea5-c4b5-4d3b-8987-e706e80728e3","error":"certificate ray-head-cert-ray-cluster is not ready at current generation 3"}
{"level":"error","ts":"2026-11-03T15:47:29.110Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"52ebcea5-c4b5-4d3b-8987-e706e80728e3","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 3","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:47:29.110Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"52ebcea5-c4b5-4d3b-8987-e706e80728e3","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 3"}
{"level":"info","ts":"2026-11-03T15:47:29.110Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"52ebcea5-c4b5-4d3b-8987-e706e80728e3","filter labels":["test-ns-4jsqd",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:47:29.110Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"52ebcea5-c4b5-4d3b-8987-e706e80728e3","filter labels":["test-ns-4jsqd",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:47:29.110Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"52ebcea5-c4b5-4d3b-8987-e706e80728e3","filter labels":["test-ns-4jsqd",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:47:29.110Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"52ebcea5-c4b5-4d3b-8987-e706e80728e3"}
{"level":"error","ts":"2026-11-03T15:47:29.110Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"52ebcea5-c4b5-4d3b-8987-e706e80728e3","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 3","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:47:30.391Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b1c53bfc-7e69-4e09-9613-27c8e1405601"}
{"level":"info","ts":"2026-11-03T15:47:30.391Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b1c53bfc-7e69-4e09-9613-27c8e1405601"}
{"level":"info","ts":"2026-11-03T15:47:30.391Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b1c53bfc-7e69-4e09-9613-27c8e1405601","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:47:30.391Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b1c53bfc-7e69-4e09-9613-27c8e1405601"}
{"level":"info","ts":"2026-11-03T15:47:30.442Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b1c53bfc-7e69-4e09-9613-27c8e1405601","error":"certificate ray-head-cert-ray-cluster is not ready at current generation 3"}
{"level":"error","ts":"2026-11-03T15:47:30.442Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b1c53bfc-7e69-4e09-9613-27c8e1405601","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 3","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:47:30.442Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b1c53bfc-7e69-4e09-9613-27c8e1405601","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 3"}
{"level":"info","ts":"2026-11-03T15:47:30.442Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b1c53bfc-7e69-4e09-9613-27c8e1405601","filter labels":["test-ns-4jsqd",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:47:30.442Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b1c53bfc-7e69-4e09-9613-27c8e1405601","filter labels":["test-ns-4jsqd",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:47:30.442Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b1c53bfc-7e69-4e09-9613-27c8e1405601","filter labels":["test-ns-4jsqd",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:47:30.442Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b1c53bfc-7e69-4e09-9613-27c8e1405601"}
{"level":"error","ts":"2026-11-03T15:47:30.442Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"b1c53bfc-7e69-4e09-9613-27c8e1405601","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 3","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:47:33.003Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"9ddc954a-cfbb-48f2-bedb-96e1671d5961"}
{"level":"info","ts":"2026-11-03T15:47:33.003Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"9ddc954a-cfbb-48f2-bedb-96e1671d5961"}
{"level":"info","ts":"2026-11-03T15:47:33.003Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"9ddc954a-cfbb-48f2-bedb-96e1671d5961","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:47:33.003Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"9ddc954a-cfbb-48f2-bedb-96e1671d5961"}
{"level":"info","ts":"2026-11-03T15:47:33.054Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"9ddc954a-cfbb-48f2-bedb-96e1671d5961","error":"certificate ray-head-cert-ray-cluster is not ready at current generation 3"}
{"level":"error","ts":"2026-11-03T15:47:33.054Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"9ddc954a-cfbb-48f2-bedb-96e1671d5961","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 3","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:47:33.054Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"9ddc954a-cfbb-48f2-bedb-96e1671d5961","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 3"}
{"level":"info","ts":"2026-11-03T15:47:33.054Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"9ddc954a-cfbb-48f2-bedb-96e1671d5961","filter labels":["test-ns-4jsqd",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:47:33.054Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"9ddc954a-cfbb-48f2-bedb-96e1671d5961","filter labels":["test-ns-4jsqd",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:47:33.054Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"9ddc954a-cfbb-48f2-bedb-96e1671d5961","filter labels":["test-ns-4jsqd",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:47:33.054Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"9ddc954a-cfbb-48f2-bedb-96e1671d5961"}
{"level":"error","ts":"2026-11-03T15:47:33.054Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"9ddc954a-cfbb-48f2-bedb-96e1671d5961","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 3","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:47:33.248Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"reconcileID":"893cc6bf-ce67-4ba8-8d4a-5497e26c27b3"}
{"level":"info","ts":"2026-11-03T15:47:33.248Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"namespace":"test-ns-4jsqd","name":"ray-cluster","reconcileID":"4b1b3a69-8d63-47a1-9f8f-4c2d7e9f5b67","namespacedName":{"name":"ray-cluster","namespace":"test-ns-4jsqd"}}
{"level":"info","ts":"2026-11-03T15:47:33.248Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"namespace":"test-ns-4jsqd","name":"ray-cluster","reconcileID":"4b1b3a69-8d63-47a1-9f8f-4c2d7e9f5b67","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:47:33.248Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"namespace":"test-ns-4jsqd","name":"ray-cluster","reconcileID":"4b1b3a69-8d63-47a1-9f8f-4c2d7e9f5b67","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:47:33.248Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"namespace":"test-ns-4jsqd","name":"ray-cluster","reconcileID":"4b1b3a69-8d63-47a1-9f8f-4c2d7e9f5b67","namespace":"test-ns-4jsqd","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-11-03T15:47:33.343Z","logger":"authentication-controller","msg":"Deleted shared ReferenceGrant (last cluster with auth)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"namespace":"test-ns-4jsqd","name":"ray-cluster","reconcileID":"4b1b3a69-8d63-47a1-9f8f-4c2d7e9f5b67","name":"kuberay-gateway-access","namespace":"test-ns-4jsqd"}
{"level":"info","ts":"2026-11-03T15:47:33.400Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"namespace":"test-ns-4jsqd","name":"ray-cluster","reconcileID":"4b1b3a69-8d63-47a1-9f8f-4c2d7e9f5b67","name":"test-ns-4jsqd-ray-cluster","namespace":"opendatahub"}
{"level":"info","ts":"2026-11-03T15:47:33.452Z","logger":"authentication-controller","msg":"Deleted ConfigMap","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"namespace":"test-ns-4jsqd","name":"ray-cluster","reconcileID":"4b1b3a69-8d63-47a1-9f8f-4c2d7e9f5b67","configMap":"kube-rbac-proxy-config-ray-cluster"}
{"level":"info","ts":"2026-11-03T15:47:33.511Z","logger":"raycluster-resource","msg":"default","name":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:47:33.511Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"ray-cluster","namespace":"test-ns-4jsqd"}
{"level":"info","ts":"2026-11-03T15:47:33.569Z","logger":"raycluster-resource","msg":"validate update","name":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:47:33.624Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"namespace":"test-ns-4jsqd","name":"ray-cluster","reconcileID":"4b1b3a69-8d63-47a1-9f8f-4c2d7e9f5b67","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:47:33.624Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"namespace":"test-ns-4jsqd","name":"ray-cluster","reconcileID":"e32c647b-bf7f-401e-b447-0fd4aa8cd576","namespacedName":{"name":"ray-cluster","namespace":"test-ns-4jsqd"}}
{"level":"info","ts":"2026-11-03T15:47:33.624Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"namespace":"test-ns-4jsqd","name":"ray-cluster","reconcileID":"e32c647b-bf7f-401e-b447-0fd4aa8cd576"}
{"level":"info","ts":"2026-11-03T15:47:33.624Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"namespace":"test-ns-4jsqd","name":"ray-cluster","reconcileID":"e32c647b-bf7f-401e-b447-0fd4aa8cd576","namespace":"test-ns-4jsqd","authClusters":0}
{"level":"info","ts":"2026-11-03T15:47:33.624Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"namespace":"test-ns-4jsqd","name":"ray-cluster","reconcileID":"e32c647b-bf7f-401e-b447-0fd4aa8cd576","namespace":"test-ns-4jsqd"}
{"level":"info","ts":"2026-11-03T15:47:33.624Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"namespace":"test-ns-4jsqd","name":"ray-cluster","reconcileID":"a02a17ef-7c27-42d8-a6e3-f91b5d70646c","namespacedName":{"name":"ray-cluster","namespace":"test-ns-4jsqd"}}
{"level":"info","ts":"2026-11-03T15:47:33.624Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"namespace":"test-ns-4jsqd","name":"ray-cluster","reconcileID":"a02a17ef-7c27-42d8-a6e3-f91b5d70646c"}
{"level":"info","ts":"2026-11-03T15:47:33.624Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"namespace":"test-ns-4jsqd","name":"ray-cluster","reconcileID":"a02a17ef-7c27-42d8-a6e3-f91b5d70646c","namespace":"test-ns-4jsqd","authClusters":0}
{"level":"info","ts":"2026-11-03T15:47:33.624Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"namespace":"test-ns-4jsqd","name":"ray-cluster","reconcileID":"a02a17ef-7c27-42d8-a6e3-f91b5d70646c","namespace":"test-ns-4jsqd"}
{"level":"info","ts":"2026-11-03T15:47:33.628Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"namespace":"test-ns-4jsqd","name":"ray-cluster","reconcileID":"a2325eca-d31c-4fe4-812f-03e953074f24","namespacedName":{"name":"ray-cluster","namespace":"test-ns-4jsqd"}}
{"level":"info","ts":"2026-11-03T15:47:33.628Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"namespace":"test-ns-4jsqd","name":"ray-cluster","reconcileID":"a2325eca-d31c-4fe4-812f-03e953074f24"}
{"level":"info","ts":"2026-11-03T15:47:33.628Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"namespace":"test-ns-4jsqd","name":"ray-cluster","reconcileID":"a2325eca-d31c-4fe4-812f-03e953074f24","namespace":"test-ns-4jsqd","authClusters":0}
{"level":"info","ts":"2026-11-03T15:47:33.628Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-4jsqd"},"namespace":"test-ns-4jsqd","name":"ray-cluster","reconcileID":"a2325eca-d31c-4fe4-812f-03e953074f24","namespace":"test-ns-4jsqd"}
{"level":"info","ts":"2026-11-03T15:48:27.238Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"5644e2e4-8ee2-4a60-8abd-09b0bb7a6248"}
{"level":"info","ts":"2026-11-03T15:48:27.238Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"5644e2e4-8ee2-4a60-8abd-09b0bb7a6248"}
{"level":"info","ts":"2026-11-03T15:48:27.238Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"5644e2e4-8ee2-4a60-8abd-09b0bb7a6248","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:48:27.238Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"5644e2e4-8ee2-4a60-8abd-09b0bb7a6248","Found 1 head Pod":"ray-cluster-head-jhq58","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:48:27.238Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"5644e2e4-8ee2-4a60-8abd-09b0bb7a6248","head Pod":"ray-cluster-head-jhq58","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-jhq58. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:48:27.238Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"5644e2e4-8ee2-4a60-8abd-09b0bb7a6248","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-11-03T15:48:27.238Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"5644e2e4-8ee2-4a60-8abd-09b0bb7a6248","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:48:27.238Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"5644e2e4-8ee2-4a60-8abd-09b0bb7a6248","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-11-03T15:48:27.238Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"5644e2e4-8ee2-4a60-8abd-09b0bb7a6248","group":"small-group","indices":{}}
{"level":"info","ts":"2026-11-03T15:48:27.238Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"5644e2e4-8ee2-4a60-8abd-09b0bb7a6248","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:48:27.238Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"5644e2e4-8ee2-4a60-8abd-09b0bb7a6248","podName":"ray-cluster-head-jhq58","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:48:27.238Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"5644e2e4-8ee2-4a60-8abd-09b0bb7a6248","podName":"ray-cluster-head-jhq58","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:48:27.238Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"5644e2e4-8ee2-4a60-8abd-09b0bb7a6248","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:48:27.238Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"5644e2e4-8ee2-4a60-8abd-09b0bb7a6248","seconds":300}
{"level":"info","ts":"2026-11-03T15:48:28.202Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"c5303b2b-b3a0-41b2-8052-36b240288b0b"}
{"level":"info","ts":"2026-11-03T15:48:28.202Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"c5303b2b-b3a0-41b2-8052-36b240288b0b"}
{"level":"info","ts":"2026-11-03T15:48:28.202Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"c5303b2b-b3a0-41b2-8052-36b240288b0b","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:48:28.203Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"c5303b2b-b3a0-41b2-8052-36b240288b0b","Found 1 head Pod":"ray-cluster-head-jhq58","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:48:28.203Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"c5303b2b-b3a0-41b2-8052-36b240288b0b","head Pod":"ray-cluster-head-jhq58","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-jhq58. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:48:28.203Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"c5303b2b-b3a0-41b2-8052-36b240288b0b","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-11-03T15:48:28.203Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"c5303b2b-b3a0-41b2-8052-36b240288b0b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:48:28.203Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"c5303b2b-b3a0-41b2-8052-36b240288b0b","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-11-03T15:48:28.203Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"c5303b2b-b3a0-41b2-8052-36b240288b0b","group":"small-group","indices":{}}
{"level":"info","ts":"2026-11-03T15:48:28.203Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"c5303b2b-b3a0-41b2-8052-36b240288b0b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:48:28.203Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"c5303b2b-b3a0-41b2-8052-36b240288b0b","podName":"ray-cluster-head-jhq58","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:46:54Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:48:28.203Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"c5303b2b-b3a0-41b2-8052-36b240288b0b","podName":"ray-cluster-head-jhq58","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:46:54Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:48:28.203Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"c5303b2b-b3a0-41b2-8052-36b240288b0b","name":"ray-cluster","old status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:46:56Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"ray-cluster-head-jhq58","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:46:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"PodInitializing","message":"containers with unready status: [ray-head autoscaler kube-rbac-proxy]; autoscaler: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:48:28Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"ray-cluster-head-jhq58","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:46:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:48:28.264Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"ea9c0d72-27dc-4c4c-bf78-eeca37480d61","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:48:30.265Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"25857805-2431-492b-a49b-468bb56774b1"}
{"level":"info","ts":"2026-11-03T15:48:30.265Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"25857805-2431-492b-a49b-468bb56774b1"}
{"level":"info","ts":"2026-11-03T15:48:30.265Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"25857805-2431-492b-a49b-468bb56774b1","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:48:30.265Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"25857805-2431-492b-a49b-468bb56774b1","Found 1 head Pod":"ray-cluster-head-jhq58","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:48:30.265Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"25857805-2431-492b-a49b-468bb56774b1","head Pod":"ray-cluster-head-jhq58","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-jhq58. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:48:30.265Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"25857805-2431-492b-a49b-468bb56774b1","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-11-03T15:48:30.265Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"25857805-2431-492b-a49b-468bb56774b1","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:48:30.265Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"25857805-2431-492b-a49b-468bb56774b1","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-11-03T15:48:30.265Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"25857805-2431-492b-a49b-468bb56774b1","group":"small-group","indices":{}}
{"level":"info","ts":"2026-11-03T15:48:30.265Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"25857805-2431-492b-a49b-468bb56774b1","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:48:30.265Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"25857805-2431-492b-a49b-468bb56774b1","podName":"ray-cluster-head-jhq58","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:46:54Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:48:30.265Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"25857805-2431-492b-a49b-468bb56774b1","podName":"ray-cluster-head-jhq58","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:46:54Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:48:30.266Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"25857805-2431-492b-a49b-468bb56774b1","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:48:30.266Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"25857805-2431-492b-a49b-468bb56774b1","seconds":300}
{"level":"info","ts":"2026-11-03T15:48:44.260Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"020d739c-82c4-4ad4-b49a-926cad79f01f"}
{"level":"info","ts":"2026-11-03T15:48:44.260Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"020d739c-82c4-4ad4-b49a-926cad79f01f"}
{"level":"info","ts":"2026-11-03T15:48:44.260Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"020d739c-82c4-4ad4-b49a-926cad79f01f","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:48:44.260Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"020d739c-82c4-4ad4-b49a-926cad79f01f","Found 1 head Pod":"ray-cluster-head-jhq58","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:48:44.260Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"020d739c-82c4-4ad4-b49a-926cad79f01f","head Pod":"ray-cluster-head-jhq58","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-jhq58. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:48:44.260Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"020d739c-82c4-4ad4-b49a-926cad79f01f","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-11-03T15:48:44.260Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"020d739c-82c4-4ad4-b49a-926cad79f01f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:48:44.260Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"020d739c-82c4-4ad4-b49a-926cad79f01f","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-11-03T15:48:44.260Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"020d739c-82c4-4ad4-b49a-926cad79f01f","group":"small-group","indices":{}}
{"level":"info","ts":"2026-11-03T15:48:44.260Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"020d739c-82c4-4ad4-b49a-926cad79f01f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:48:44.261Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"020d739c-82c4-4ad4-b49a-926cad79f01f","name":"ray-cluster","old status":{"desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:48:28Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"ray-cluster-head-jhq58","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:46:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:48:44Z","stateTransitionTimes":{"ready":"2026-11-03T15:48:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"ray-cluster-head-jhq58","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:46:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:48:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:48:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:48:44.322Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"58a7affc-3a3d-4c50-9b49-aa53ea0d85e2","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:48:46.322Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"8d1999a6-576f-4c01-9816-f83f689c401c"}
{"level":"info","ts":"2026-11-03T15:48:46.322Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"8d1999a6-576f-4c01-9816-f83f689c401c"}
{"level":"info","ts":"2026-11-03T15:48:46.322Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"8d1999a6-576f-4c01-9816-f83f689c401c","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:48:46.322Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"8d1999a6-576f-4c01-9816-f83f689c401c","Found 1 head Pod":"ray-cluster-head-jhq58","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:48:46.322Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"8d1999a6-576f-4c01-9816-f83f689c401c","head Pod":"ray-cluster-head-jhq58","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-jhq58. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:48:46.322Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"8d1999a6-576f-4c01-9816-f83f689c401c","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-11-03T15:48:46.322Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"8d1999a6-576f-4c01-9816-f83f689c401c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:48:46.322Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"8d1999a6-576f-4c01-9816-f83f689c401c","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-11-03T15:48:46.322Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"8d1999a6-576f-4c01-9816-f83f689c401c","group":"small-group","indices":{}}
{"level":"info","ts":"2026-11-03T15:48:46.322Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"8d1999a6-576f-4c01-9816-f83f689c401c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:48:46.322Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"8d1999a6-576f-4c01-9816-f83f689c401c","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:48:46.322Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"8d1999a6-576f-4c01-9816-f83f689c401c","seconds":300}
{"level":"info","ts":"2026-11-03T15:48:52.354Z","logger":"raycluster-resource","msg":"default","name":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:48:52.354Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"ray-cluster","namespace":"test-ns-zjpgs"}
{"level":"info","ts":"2026-11-03T15:48:52.412Z","logger":"raycluster-resource","msg":"validate update","name":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:48:52.467Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"dc97c660-ec52-47e2-8252-295d4976bfa1"}
{"level":"info","ts":"2026-11-03T15:48:52.467Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"e8e73a64-0152-45a2-8cc5-33b17152ada8","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:48:52.467Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"namespace":"test-ns-zjpgs","name":"ray-cluster","reconcileID":"9be751ea-98d4-4af2-ad57-910178eb0cff","namespacedName":{"name":"ray-cluster","namespace":"test-ns-zjpgs"}}
{"level":"info","ts":"2026-11-03T15:48:52.467Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"namespace":"test-ns-zjpgs","name":"ray-cluster","reconcileID":"9be751ea-98d4-4af2-ad57-910178eb0cff","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:48:52.467Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"namespace":"test-ns-zjpgs","name":"ray-cluster","reconcileID":"9be751ea-98d4-4af2-ad57-910178eb0cff","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:48:52.467Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"dc97c660-ec52-47e2-8252-295d4976bfa1"}
{"level":"info","ts":"2026-11-03T15:48:52.467Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"dc97c660-ec52-47e2-8252-295d4976bfa1","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:48:52.467Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"dc97c660-ec52-47e2-8252-295d4976bfa1","Found 1 head Pod":"ray-cluster-head-jhq58","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:48:52.467Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"dc97c660-ec52-47e2-8252-295d4976bfa1","head Pod":"ray-cluster-head-jhq58","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-jhq58. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:48:52.467Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"dc97c660-ec52-47e2-8252-295d4976bfa1","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":1}
{"level":"info","ts":"2026-11-03T15:48:52.467Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"dc97c660-ec52-47e2-8252-295d4976bfa1","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:48:52.467Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"dc97c660-ec52-47e2-8252-295d4976bfa1","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-11-03T15:48:52.467Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"dc97c660-ec52-47e2-8252-295d4976bfa1","group":"small-group","indices":{}}
{"level":"info","ts":"2026-11-03T15:48:52.467Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"dc97c660-ec52-47e2-8252-295d4976bfa1","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-11-03T15:48:52.467Z","logger":"controllers.NetworkPolicy","msg":"Reconciling NetworkPolicies for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"5a887add-def2-4849-bcaf-91e8989224e7","cluster":"ray-cluster","namespace":"test-ns-zjpgs"}
{"level":"info","ts":"2026-11-03T15:48:52.563Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"namespace":"test-ns-zjpgs","name":"ray-cluster","reconcileID":"9be751ea-98d4-4af2-ad57-910178eb0cff","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:48:52.567Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"dc97c660-ec52-47e2-8252-295d4976bfa1","creating worker for group":"small-group","index":0,"total":1,"replicaIndex":0}
{"level":"info","ts":"2026-11-03T15:48:52.567Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"dc97c660-ec52-47e2-8252-295d4976bfa1","nodeType":"worker","rayStartParams":{"address":"ray-cluster-head-svc.test-ns-zjpgs.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"1G"},"requests":{"cpu":"1","memory":"1G"}}}
{"level":"info","ts":"2026-11-03T15:48:52.567Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"dc97c660-ec52-47e2-8252-295d4976bfa1","rayStartCmd":"ray start  --address=ray-cluster-head-svc.test-ns-zjpgs.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=1000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-11-03T15:48:52.567Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"dc97c660-ec52-47e2-8252-295d4976bfa1","rayNodeType":"worker","generatedCmd":"ulimit -n ${RAY_START_ULIMIT_OPEN_FILES:-65536}; ray start  --address=ray-cluster-head-svc.test-ns-zjpgs.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=1000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-11-03T15:48:52.567Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"dc97c660-ec52-47e2-8252-295d4976bfa1","enabled":true}
{"level":"info","ts":"2026-11-03T15:48:52.626Z","logger":"controllers.NetworkPolicy","msg":"Successfully reconciled NetworkPolicies for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"5a887add-def2-4849-bcaf-91e8989224e7","cluster":"ray-cluster","namespace":"test-ns-zjpgs"}
{"level":"info","ts":"2026-11-03T15:48:52.638Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"dc97c660-ec52-47e2-8252-295d4976bfa1","name":"ray-cluster-small-group-worker-xbgwc"}
{"level":"info","ts":"2026-11-03T15:48:52.638Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"dc97c660-ec52-47e2-8252-295d4976bfa1","podName":"ray-cluster-small-group-worker-xbgwc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:48:52.661Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"dc97c660-ec52-47e2-8252-295d4976bfa1","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:48:44Z","stateTransitionTimes":{"ready":"2026-11-03T15:48:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"ray-cluster-head-jhq58","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:46:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:48:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:48:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:48:52Z","stateTransitionTimes":{"ready":"2026-11-03T15:48:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"ray-cluster-head-jhq58","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:46:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:48:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:48:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":2}}
{"level":"info","ts":"2026-11-03T15:48:52.716Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"dc97c660-ec52-47e2-8252-295d4976bfa1","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-zjpgs","name":"ray-cluster"}}
{"level":"info","ts":"2026-11-03T15:48:52.716Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"dc97c660-ec52-47e2-8252-295d4976bfa1"}
{"level":"error","ts":"2026-11-03T15:48:52.716Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"dc97c660-ec52-47e2-8252-295d4976bfa1","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:48:52.716Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9bd77404-7c3d-400b-822c-fe9733e348c5"}
{"level":"info","ts":"2026-11-03T15:48:52.716Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9bd77404-7c3d-400b-822c-fe9733e348c5"}
{"level":"info","ts":"2026-11-03T15:48:52.716Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9bd77404-7c3d-400b-822c-fe9733e348c5","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:48:52.716Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9bd77404-7c3d-400b-822c-fe9733e348c5","Found 1 head Pod":"ray-cluster-head-jhq58","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:48:52.716Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9bd77404-7c3d-400b-822c-fe9733e348c5","head Pod":"ray-cluster-head-jhq58","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-jhq58. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:48:52.716Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9bd77404-7c3d-400b-822c-fe9733e348c5","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":1}
{"level":"info","ts":"2026-11-03T15:48:52.716Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9bd77404-7c3d-400b-822c-fe9733e348c5","worker Pod":"ray-cluster-small-group-worker-xbgwc","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-xbgwc. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:48:52.716Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9bd77404-7c3d-400b-822c-fe9733e348c5","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:48:52.716Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9bd77404-7c3d-400b-822c-fe9733e348c5","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:48:52.716Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9bd77404-7c3d-400b-822c-fe9733e348c5","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:48:52.716Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9bd77404-7c3d-400b-822c-fe9733e348c5","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:48:52.716Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9bd77404-7c3d-400b-822c-fe9733e348c5","podName":"ray-cluster-small-group-worker-xbgwc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:48:52.716Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9bd77404-7c3d-400b-822c-fe9733e348c5","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:48:44Z","stateTransitionTimes":{"ready":"2026-11-03T15:48:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"ray-cluster-head-jhq58","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":2,"lastTransitionTime":"2026-11-03T15:46:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:48:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:48:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:48:52Z","stateTransitionTimes":{"ready":"2026-11-03T15:48:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"ray-cluster-head-jhq58","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":2,"lastTransitionTime":"2026-11-03T15:46:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:48:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:48:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":2}}
{"level":"info","ts":"2026-11-03T15:48:52.777Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"27cfb282-95e1-4e59-b397-9bd6be247f89"}
{"level":"info","ts":"2026-11-03T15:48:52.777Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"27cfb282-95e1-4e59-b397-9bd6be247f89"}
{"level":"info","ts":"2026-11-03T15:48:52.777Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"27cfb282-95e1-4e59-b397-9bd6be247f89","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:48:52.777Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"27cfb282-95e1-4e59-b397-9bd6be247f89","Found 1 head Pod":"ray-cluster-head-jhq58","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:48:52.777Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"27cfb282-95e1-4e59-b397-9bd6be247f89","head Pod":"ray-cluster-head-jhq58","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-jhq58. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:48:52.777Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"27cfb282-95e1-4e59-b397-9bd6be247f89","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":1}
{"level":"info","ts":"2026-11-03T15:48:52.777Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"27cfb282-95e1-4e59-b397-9bd6be247f89","worker Pod":"ray-cluster-small-group-worker-xbgwc","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-xbgwc. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:48:52.777Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"27cfb282-95e1-4e59-b397-9bd6be247f89","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:48:52.777Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"27cfb282-95e1-4e59-b397-9bd6be247f89","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:48:52.777Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"27cfb282-95e1-4e59-b397-9bd6be247f89","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:48:52.777Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"27cfb282-95e1-4e59-b397-9bd6be247f89","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:48:52.777Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"27cfb282-95e1-4e59-b397-9bd6be247f89","podName":"ray-cluster-small-group-worker-xbgwc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:48:52.777Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"27cfb282-95e1-4e59-b397-9bd6be247f89","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:48:44Z","stateTransitionTimes":{"ready":"2026-11-03T15:48:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"ray-cluster-head-jhq58","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":2,"lastTransitionTime":"2026-11-03T15:46:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:48:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:48:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:48:52Z","stateTransitionTimes":{"ready":"2026-11-03T15:48:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"ray-cluster-head-jhq58","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":2,"lastTransitionTime":"2026-11-03T15:46:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:48:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:48:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":2}}
{"level":"info","ts":"2026-11-03T15:48:52.832Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"27cfb282-95e1-4e59-b397-9bd6be247f89","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-zjpgs","name":"ray-cluster"}}
{"level":"info","ts":"2026-11-03T15:48:52.832Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"27cfb282-95e1-4e59-b397-9bd6be247f89"}
{"level":"error","ts":"2026-11-03T15:48:52.832Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"27cfb282-95e1-4e59-b397-9bd6be247f89","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:48:52.837Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"bbe8914f-0f2d-4ed2-a38f-a5c539289599"}
{"level":"info","ts":"2026-11-03T15:48:52.837Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"bbe8914f-0f2d-4ed2-a38f-a5c539289599"}
{"level":"info","ts":"2026-11-03T15:48:52.837Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"bbe8914f-0f2d-4ed2-a38f-a5c539289599","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:48:52.838Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"bbe8914f-0f2d-4ed2-a38f-a5c539289599","Found 1 head Pod":"ray-cluster-head-jhq58","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:48:52.838Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"bbe8914f-0f2d-4ed2-a38f-a5c539289599","head Pod":"ray-cluster-head-jhq58","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-jhq58. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:48:52.838Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"bbe8914f-0f2d-4ed2-a38f-a5c539289599","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":1}
{"level":"info","ts":"2026-11-03T15:48:52.838Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"bbe8914f-0f2d-4ed2-a38f-a5c539289599","worker Pod":"ray-cluster-small-group-worker-xbgwc","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-xbgwc. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:48:52.838Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"bbe8914f-0f2d-4ed2-a38f-a5c539289599","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:48:52.838Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"bbe8914f-0f2d-4ed2-a38f-a5c539289599","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:48:52.838Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"bbe8914f-0f2d-4ed2-a38f-a5c539289599","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:48:52.838Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"bbe8914f-0f2d-4ed2-a38f-a5c539289599","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:48:52.838Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"bbe8914f-0f2d-4ed2-a38f-a5c539289599","podName":"ray-cluster-small-group-worker-xbgwc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:48:52.838Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"bbe8914f-0f2d-4ed2-a38f-a5c539289599","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:48:52.838Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"bbe8914f-0f2d-4ed2-a38f-a5c539289599","seconds":300}
{"level":"info","ts":"2026-11-03T15:48:53.273Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"b6a1e1a6-8c0b-4a4e-8c4c-d99eabc43a1f","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:48:53.391Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"216ef174-8148-4dff-a299-36a681070047"}
{"level":"info","ts":"2026-11-03T15:48:53.391Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"216ef174-8148-4dff-a299-36a681070047"}
{"level":"info","ts":"2026-11-03T15:48:53.391Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"216ef174-8148-4dff-a299-36a681070047","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:48:53.391Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"216ef174-8148-4dff-a299-36a681070047","Found 1 head Pod":"ray-cluster-head-jhq58","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:48:53.391Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"216ef174-8148-4dff-a299-36a681070047","head Pod":"ray-cluster-head-jhq58","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-jhq58. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:48:53.391Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"216ef174-8148-4dff-a299-36a681070047","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":1}
{"level":"info","ts":"2026-11-03T15:48:53.391Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"216ef174-8148-4dff-a299-36a681070047","worker Pod":"ray-cluster-small-group-worker-xbgwc","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-xbgwc. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:48:53.391Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"216ef174-8148-4dff-a299-36a681070047","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:48:53.391Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"216ef174-8148-4dff-a299-36a681070047","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:48:53.391Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"216ef174-8148-4dff-a299-36a681070047","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:48:53.391Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"216ef174-8148-4dff-a299-36a681070047","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:48:53.392Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"216ef174-8148-4dff-a299-36a681070047","podName":"ray-cluster-small-group-worker-xbgwc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:48:53.392Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"216ef174-8148-4dff-a299-36a681070047","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:48:53.392Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"216ef174-8148-4dff-a299-36a681070047","seconds":300}
{"level":"info","ts":"2026-11-03T15:48:54.606Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"b82dc6a7-d7fe-4327-8e2d-e1561292ab88"}
{"level":"info","ts":"2026-11-03T15:48:54.606Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"b82dc6a7-d7fe-4327-8e2d-e1561292ab88"}
{"level":"info","ts":"2026-11-03T15:48:54.606Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"b82dc6a7-d7fe-4327-8e2d-e1561292ab88","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:48:54.606Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"b82dc6a7-d7fe-4327-8e2d-e1561292ab88","Found 1 head Pod":"ray-cluster-head-jhq58","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:48:54.606Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"b82dc6a7-d7fe-4327-8e2d-e1561292ab88","head Pod":"ray-cluster-head-jhq58","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-jhq58. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:48:54.606Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"b82dc6a7-d7fe-4327-8e2d-e1561292ab88","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":1}
{"level":"info","ts":"2026-11-03T15:48:54.606Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"b82dc6a7-d7fe-4327-8e2d-e1561292ab88","worker Pod":"ray-cluster-small-group-worker-xbgwc","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-xbgwc. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:48:54.606Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"b82dc6a7-d7fe-4327-8e2d-e1561292ab88","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:48:54.606Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"b82dc6a7-d7fe-4327-8e2d-e1561292ab88","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:48:54.606Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"b82dc6a7-d7fe-4327-8e2d-e1561292ab88","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:48:54.606Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"b82dc6a7-d7fe-4327-8e2d-e1561292ab88","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:48:54.606Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"b82dc6a7-d7fe-4327-8e2d-e1561292ab88","podName":"ray-cluster-small-group-worker-xbgwc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:48:54.607Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"b82dc6a7-d7fe-4327-8e2d-e1561292ab88","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:48:54.607Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"b82dc6a7-d7fe-4327-8e2d-e1561292ab88","seconds":300}
{"level":"info","ts":"2026-11-03T15:48:59.038Z","logger":"raycluster-resource","msg":"default","name":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:48:59.038Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"ray-cluster","namespace":"test-ns-zjpgs"}
{"level":"info","ts":"2026-11-03T15:48:59.096Z","logger":"raycluster-resource","msg":"validate update","name":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:48:59.150Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"namespace":"test-ns-zjpgs","name":"ray-cluster","reconcileID":"bbb5b923-7826-4832-b1aa-17c181036122","namespacedName":{"name":"ray-cluster","namespace":"test-ns-zjpgs"}}
{"level":"info","ts":"2026-11-03T15:48:59.151Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"namespace":"test-ns-zjpgs","name":"ray-cluster","reconcileID":"bbb5b923-7826-4832-b1aa-17c181036122","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:48:59.151Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"namespace":"test-ns-zjpgs","name":"ray-cluster","reconcileID":"bbb5b923-7826-4832-b1aa-17c181036122","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:48:59.151Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9992c9d9-fa35-494c-b4b6-da58218b6eb9"}
{"level":"info","ts":"2026-11-03T15:48:59.151Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9992c9d9-fa35-494c-b4b6-da58218b6eb9"}
{"level":"info","ts":"2026-11-03T15:48:59.151Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9992c9d9-fa35-494c-b4b6-da58218b6eb9","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:48:59.151Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9992c9d9-fa35-494c-b4b6-da58218b6eb9","Found 1 head Pod":"ray-cluster-head-jhq58","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:48:59.151Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9992c9d9-fa35-494c-b4b6-da58218b6eb9","head Pod":"ray-cluster-head-jhq58","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-jhq58. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:48:59.151Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9992c9d9-fa35-494c-b4b6-da58218b6eb9","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-11-03T15:48:59.151Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9992c9d9-fa35-494c-b4b6-da58218b6eb9","worker Pod":"ray-cluster-small-group-worker-xbgwc","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-xbgwc. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:48:59.151Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9992c9d9-fa35-494c-b4b6-da58218b6eb9","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:48:59.151Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9992c9d9-fa35-494c-b4b6-da58218b6eb9","workerReplicas":2,"NumOfHosts":1,"runningPods":1,"diff":1}
{"level":"info","ts":"2026-11-03T15:48:59.151Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9992c9d9-fa35-494c-b4b6-da58218b6eb9","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:48:59.151Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9992c9d9-fa35-494c-b4b6-da58218b6eb9","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-11-03T15:48:59.151Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"0d009ac7-ed09-4184-a178-65f143355767","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:48:59.151Z","logger":"controllers.NetworkPolicy","msg":"Reconciling NetworkPolicies for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"19f0225b-219b-4c67-8d05-5aba78778360","cluster":"ray-cluster","namespace":"test-ns-zjpgs"}
{"level":"info","ts":"2026-11-03T15:48:59.249Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"namespace":"test-ns-zjpgs","name":"ray-cluster","reconcileID":"bbb5b923-7826-4832-b1aa-17c181036122","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:48:59.251Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9992c9d9-fa35-494c-b4b6-da58218b6eb9","creating worker for group":"small-group","index":0,"total":1,"replicaIndex":1}
{"level":"info","ts":"2026-11-03T15:48:59.251Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9992c9d9-fa35-494c-b4b6-da58218b6eb9","nodeType":"worker","rayStartParams":{"address":"ray-cluster-head-svc.test-ns-zjpgs.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"1G"},"requests":{"cpu":"1","memory":"1G"}}}
{"level":"info","ts":"2026-11-03T15:48:59.251Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9992c9d9-fa35-494c-b4b6-da58218b6eb9","rayStartCmd":"ray start  --address=ray-cluster-head-svc.test-ns-zjpgs.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=1000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-11-03T15:48:59.252Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9992c9d9-fa35-494c-b4b6-da58218b6eb9","rayNodeType":"worker","generatedCmd":"ulimit -n ${RAY_START_ULIMIT_OPEN_FILES:-65536}; ray start  --address=ray-cluster-head-svc.test-ns-zjpgs.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=1000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-11-03T15:48:59.252Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9992c9d9-fa35-494c-b4b6-da58218b6eb9","enabled":true}
{"level":"info","ts":"2026-11-03T15:48:59.313Z","logger":"controllers.NetworkPolicy","msg":"Successfully reconciled NetworkPolicies for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"19f0225b-219b-4c67-8d05-5aba78778360","cluster":"ray-cluster","namespace":"test-ns-zjpgs"}
{"level":"info","ts":"2026-11-03T15:48:59.313Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9992c9d9-fa35-494c-b4b6-da58218b6eb9","name":"ray-cluster-small-group-worker-lc796"}
{"level":"info","ts":"2026-11-03T15:48:59.314Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9992c9d9-fa35-494c-b4b6-da58218b6eb9","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:48:52Z","stateTransitionTimes":{"ready":"2026-11-03T15:48:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"ray-cluster-head-jhq58","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":2,"lastTransitionTime":"2026-11-03T15:46:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:48:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:48:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":2},"new status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:48:59Z","stateTransitionTimes":{"ready":"2026-11-03T15:48:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"ray-cluster-head-jhq58","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":2,"lastTransitionTime":"2026-11-03T15:46:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:48:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:48:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3}}
{"level":"info","ts":"2026-11-03T15:48:59.368Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9992c9d9-fa35-494c-b4b6-da58218b6eb9","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-zjpgs","name":"ray-cluster"}}
{"level":"info","ts":"2026-11-03T15:48:59.368Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9992c9d9-fa35-494c-b4b6-da58218b6eb9"}
{"level":"error","ts":"2026-11-03T15:48:59.368Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9992c9d9-fa35-494c-b4b6-da58218b6eb9","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:48:59.368Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"30d9c23d-257e-40ae-8663-88a158aac0c1"}
{"level":"info","ts":"2026-11-03T15:48:59.368Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"30d9c23d-257e-40ae-8663-88a158aac0c1"}
{"level":"info","ts":"2026-11-03T15:48:59.368Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"30d9c23d-257e-40ae-8663-88a158aac0c1","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:48:59.369Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"30d9c23d-257e-40ae-8663-88a158aac0c1","Found 1 head Pod":"ray-cluster-head-jhq58","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:48:59.369Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"30d9c23d-257e-40ae-8663-88a158aac0c1","head Pod":"ray-cluster-head-jhq58","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-jhq58. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:48:59.369Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"30d9c23d-257e-40ae-8663-88a158aac0c1","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-11-03T15:48:59.369Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"30d9c23d-257e-40ae-8663-88a158aac0c1","worker Pod":"ray-cluster-small-group-worker-xbgwc","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-xbgwc. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:48:59.369Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"30d9c23d-257e-40ae-8663-88a158aac0c1","worker Pod":"ray-cluster-small-group-worker-lc796","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-lc796. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:48:59.369Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"30d9c23d-257e-40ae-8663-88a158aac0c1","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:48:59.369Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"30d9c23d-257e-40ae-8663-88a158aac0c1","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-11-03T15:48:59.369Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"30d9c23d-257e-40ae-8663-88a158aac0c1","group":"small-group","indices":{"0":true,"1":true}}
{"level":"info","ts":"2026-11-03T15:48:59.369Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"30d9c23d-257e-40ae-8663-88a158aac0c1","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:48:59.369Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"30d9c23d-257e-40ae-8663-88a158aac0c1","podName":"ray-cluster-small-group-worker-lc796","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:48:59.369Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"30d9c23d-257e-40ae-8663-88a158aac0c1","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:48:52Z","stateTransitionTimes":{"ready":"2026-11-03T15:48:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"ray-cluster-head-jhq58","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-11-03T15:46:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:48:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:48:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":2},"new status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:48:59Z","stateTransitionTimes":{"ready":"2026-11-03T15:48:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"ray-cluster-head-jhq58","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-11-03T15:46:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:48:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:48:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3}}
{"level":"info","ts":"2026-11-03T15:48:59.431Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"a373eb1b-ccc5-487f-8d1a-55edbf423809"}
{"level":"info","ts":"2026-11-03T15:48:59.431Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"a373eb1b-ccc5-487f-8d1a-55edbf423809"}
{"level":"info","ts":"2026-11-03T15:48:59.431Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"a373eb1b-ccc5-487f-8d1a-55edbf423809","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:48:59.431Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"a373eb1b-ccc5-487f-8d1a-55edbf423809","Found 1 head Pod":"ray-cluster-head-jhq58","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:48:59.431Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"a373eb1b-ccc5-487f-8d1a-55edbf423809","head Pod":"ray-cluster-head-jhq58","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-jhq58. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:48:59.431Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"a373eb1b-ccc5-487f-8d1a-55edbf423809","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-11-03T15:48:59.431Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"a373eb1b-ccc5-487f-8d1a-55edbf423809","worker Pod":"ray-cluster-small-group-worker-xbgwc","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-xbgwc. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:48:59.431Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"a373eb1b-ccc5-487f-8d1a-55edbf423809","worker Pod":"ray-cluster-small-group-worker-lc796","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-lc796. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:48:59.431Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"a373eb1b-ccc5-487f-8d1a-55edbf423809","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:48:59.431Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"a373eb1b-ccc5-487f-8d1a-55edbf423809","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-11-03T15:48:59.431Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"a373eb1b-ccc5-487f-8d1a-55edbf423809","group":"small-group","indices":{"0":true,"1":true}}
{"level":"info","ts":"2026-11-03T15:48:59.431Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"a373eb1b-ccc5-487f-8d1a-55edbf423809","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:48:59.431Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"a373eb1b-ccc5-487f-8d1a-55edbf423809","podName":"ray-cluster-small-group-worker-xbgwc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:48:59.431Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"a373eb1b-ccc5-487f-8d1a-55edbf423809","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:48:59.431Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"a373eb1b-ccc5-487f-8d1a-55edbf423809","seconds":300}
{"level":"info","ts":"2026-11-03T15:48:59.446Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4175bf1e-afeb-496d-8382-980df4f6d77d"}
{"level":"info","ts":"2026-11-03T15:48:59.446Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4175bf1e-afeb-496d-8382-980df4f6d77d"}
{"level":"info","ts":"2026-11-03T15:48:59.446Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4175bf1e-afeb-496d-8382-980df4f6d77d","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:48:59.447Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4175bf1e-afeb-496d-8382-980df4f6d77d","Found 1 head Pod":"ray-cluster-head-jhq58","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:48:59.447Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4175bf1e-afeb-496d-8382-980df4f6d77d","head Pod":"ray-cluster-head-jhq58","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-jhq58. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:48:59.447Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4175bf1e-afeb-496d-8382-980df4f6d77d","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-11-03T15:48:59.447Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4175bf1e-afeb-496d-8382-980df4f6d77d","worker Pod":"ray-cluster-small-group-worker-xbgwc","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-xbgwc. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:48:59.447Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4175bf1e-afeb-496d-8382-980df4f6d77d","worker Pod":"ray-cluster-small-group-worker-lc796","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-lc796. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:48:59.447Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4175bf1e-afeb-496d-8382-980df4f6d77d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:48:59.447Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4175bf1e-afeb-496d-8382-980df4f6d77d","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-11-03T15:48:59.447Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4175bf1e-afeb-496d-8382-980df4f6d77d","group":"small-group","indices":{"0":true,"1":true}}
{"level":"info","ts":"2026-11-03T15:48:59.447Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4175bf1e-afeb-496d-8382-980df4f6d77d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:48:59.447Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4175bf1e-afeb-496d-8382-980df4f6d77d","podName":"ray-cluster-small-group-worker-xbgwc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:48:59.447Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4175bf1e-afeb-496d-8382-980df4f6d77d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:48:59.447Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4175bf1e-afeb-496d-8382-980df4f6d77d","seconds":300}
{"level":"info","ts":"2026-11-03T15:48:59.606Z","logger":"controllers.RayClusterMTLS","msg":"Updating worker certificate","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"0d009ac7-ed09-4184-a178-65f143355767","certificate":"ray-worker-cert-ray-cluster","ipAddresses":["10.134.0.57","127.0.0.1"]}
{"level":"info","ts":"2026-11-03T15:48:59.924Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"0d009ac7-ed09-4184-a178-65f143355767","certificate":"ray-worker-cert-ray-cluster"}
{"level":"info","ts":"2026-11-03T15:48:59.924Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"0d009ac7-ed09-4184-a178-65f143355767"}
{"level":"info","ts":"2026-11-03T15:48:59.924Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"e2f43c96-9449-4ba2-bdfa-1cad7d3453e7","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:49:00.370Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9e12de56-b04c-4608-bf08-d4680c6285c6"}
{"level":"info","ts":"2026-11-03T15:49:00.370Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9e12de56-b04c-4608-bf08-d4680c6285c6"}
{"level":"info","ts":"2026-11-03T15:49:00.370Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9e12de56-b04c-4608-bf08-d4680c6285c6","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:49:00.371Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9e12de56-b04c-4608-bf08-d4680c6285c6","Found 1 head Pod":"ray-cluster-head-jhq58","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:49:00.371Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9e12de56-b04c-4608-bf08-d4680c6285c6","head Pod":"ray-cluster-head-jhq58","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-jhq58. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:49:00.371Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9e12de56-b04c-4608-bf08-d4680c6285c6","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-11-03T15:49:00.371Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9e12de56-b04c-4608-bf08-d4680c6285c6","worker Pod":"ray-cluster-small-group-worker-xbgwc","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-xbgwc. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:49:00.371Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9e12de56-b04c-4608-bf08-d4680c6285c6","worker Pod":"ray-cluster-small-group-worker-lc796","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-lc796. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:49:00.371Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9e12de56-b04c-4608-bf08-d4680c6285c6","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:49:00.371Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9e12de56-b04c-4608-bf08-d4680c6285c6","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-11-03T15:49:00.371Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9e12de56-b04c-4608-bf08-d4680c6285c6","group":"small-group","indices":{"0":true,"1":true}}
{"level":"info","ts":"2026-11-03T15:49:00.371Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9e12de56-b04c-4608-bf08-d4680c6285c6","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:49:00.371Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9e12de56-b04c-4608-bf08-d4680c6285c6","podName":"ray-cluster-small-group-worker-xbgwc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:49:00.371Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9e12de56-b04c-4608-bf08-d4680c6285c6","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:49:00.371Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9e12de56-b04c-4608-bf08-d4680c6285c6","seconds":300}
{"level":"info","ts":"2026-11-03T15:49:00.583Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"103e6b7c-1e2d-4b1f-81f4-96574d0e67bc"}
{"level":"info","ts":"2026-11-03T15:49:00.583Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"103e6b7c-1e2d-4b1f-81f4-96574d0e67bc"}
{"level":"info","ts":"2026-11-03T15:49:00.583Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"103e6b7c-1e2d-4b1f-81f4-96574d0e67bc","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:49:00.583Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"103e6b7c-1e2d-4b1f-81f4-96574d0e67bc","Found 1 head Pod":"ray-cluster-head-jhq58","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:49:00.584Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"103e6b7c-1e2d-4b1f-81f4-96574d0e67bc","head Pod":"ray-cluster-head-jhq58","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-jhq58. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:49:00.584Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"103e6b7c-1e2d-4b1f-81f4-96574d0e67bc","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-11-03T15:49:00.584Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"103e6b7c-1e2d-4b1f-81f4-96574d0e67bc","worker Pod":"ray-cluster-small-group-worker-xbgwc","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-xbgwc. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:49:00.584Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"103e6b7c-1e2d-4b1f-81f4-96574d0e67bc","worker Pod":"ray-cluster-small-group-worker-lc796","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-lc796. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:49:00.584Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"103e6b7c-1e2d-4b1f-81f4-96574d0e67bc","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:49:00.584Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"103e6b7c-1e2d-4b1f-81f4-96574d0e67bc","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-11-03T15:49:00.584Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"103e6b7c-1e2d-4b1f-81f4-96574d0e67bc","group":"small-group","indices":{"0":true,"1":true}}
{"level":"info","ts":"2026-11-03T15:49:00.584Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"103e6b7c-1e2d-4b1f-81f4-96574d0e67bc","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:49:00.584Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"103e6b7c-1e2d-4b1f-81f4-96574d0e67bc","podName":"ray-cluster-small-group-worker-xbgwc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:49:00.584Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"103e6b7c-1e2d-4b1f-81f4-96574d0e67bc","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:49:00.584Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"103e6b7c-1e2d-4b1f-81f4-96574d0e67bc","seconds":300}
{"level":"info","ts":"2026-11-03T15:49:00.586Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"e2f43c96-9449-4ba2-bdfa-1cad7d3453e7","certificate":"ray-worker-cert-ray-cluster"}
{"level":"info","ts":"2026-11-03T15:49:00.586Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"e2f43c96-9449-4ba2-bdfa-1cad7d3453e7"}
{"level":"info","ts":"2026-11-03T15:49:30.587Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"cf713c7c-8dd5-482a-9c44-2a38771516e3","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:49:31.043Z","logger":"controllers.RayClusterMTLS","msg":"Updating worker certificate","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"cf713c7c-8dd5-482a-9c44-2a38771516e3","certificate":"ray-worker-cert-ray-cluster","ipAddresses":["10.134.0.57","10.134.0.58","127.0.0.1"]}
{"level":"info","ts":"2026-11-03T15:49:31.361Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"cf713c7c-8dd5-482a-9c44-2a38771516e3","certificate":"ray-worker-cert-ray-cluster"}
{"level":"info","ts":"2026-11-03T15:49:31.361Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"cf713c7c-8dd5-482a-9c44-2a38771516e3"}
{"level":"info","ts":"2026-11-03T15:50:01.362Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"91647a0c-2522-43c5-bcf6-dcbeb919a022","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:50:10.934Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"c4ba0b94-dd1f-4fc4-823a-79457343757a"}
{"level":"info","ts":"2026-11-03T15:50:10.934Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"c4ba0b94-dd1f-4fc4-823a-79457343757a"}
{"level":"info","ts":"2026-11-03T15:50:10.934Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"c4ba0b94-dd1f-4fc4-823a-79457343757a","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:50:10.934Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"c4ba0b94-dd1f-4fc4-823a-79457343757a","Found 1 head Pod":"ray-cluster-head-jhq58","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:50:10.934Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"c4ba0b94-dd1f-4fc4-823a-79457343757a","head Pod":"ray-cluster-head-jhq58","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-jhq58. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:50:10.934Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"c4ba0b94-dd1f-4fc4-823a-79457343757a","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-11-03T15:50:10.934Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"c4ba0b94-dd1f-4fc4-823a-79457343757a","worker Pod":"ray-cluster-small-group-worker-xbgwc","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-xbgwc. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:50:10.934Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"c4ba0b94-dd1f-4fc4-823a-79457343757a","worker Pod":"ray-cluster-small-group-worker-lc796","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-lc796. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:50:10.934Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"c4ba0b94-dd1f-4fc4-823a-79457343757a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:50:10.934Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"c4ba0b94-dd1f-4fc4-823a-79457343757a","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-11-03T15:50:10.934Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"c4ba0b94-dd1f-4fc4-823a-79457343757a","group":"small-group","indices":{"0":true,"1":true}}
{"level":"info","ts":"2026-11-03T15:50:10.934Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"c4ba0b94-dd1f-4fc4-823a-79457343757a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:50:10.934Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"c4ba0b94-dd1f-4fc4-823a-79457343757a","podName":"ray-cluster-small-group-worker-xbgwc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:50:10.935Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"c4ba0b94-dd1f-4fc4-823a-79457343757a","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:50:10.935Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"c4ba0b94-dd1f-4fc4-823a-79457343757a","seconds":300}
{"level":"info","ts":"2026-11-03T15:50:11.887Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"276628b3-bba1-4e0b-9d65-51c4b101fb5d"}
{"level":"info","ts":"2026-11-03T15:50:11.887Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"276628b3-bba1-4e0b-9d65-51c4b101fb5d"}
{"level":"info","ts":"2026-11-03T15:50:11.887Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"276628b3-bba1-4e0b-9d65-51c4b101fb5d","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:50:11.887Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"276628b3-bba1-4e0b-9d65-51c4b101fb5d","Found 1 head Pod":"ray-cluster-head-jhq58","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:50:11.887Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"276628b3-bba1-4e0b-9d65-51c4b101fb5d","head Pod":"ray-cluster-head-jhq58","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-jhq58. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:50:11.887Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"276628b3-bba1-4e0b-9d65-51c4b101fb5d","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-11-03T15:50:11.887Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"276628b3-bba1-4e0b-9d65-51c4b101fb5d","worker Pod":"ray-cluster-small-group-worker-xbgwc","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-xbgwc. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:50:11.887Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"276628b3-bba1-4e0b-9d65-51c4b101fb5d","worker Pod":"ray-cluster-small-group-worker-lc796","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-lc796. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:50:11.887Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"276628b3-bba1-4e0b-9d65-51c4b101fb5d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:50:11.887Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"276628b3-bba1-4e0b-9d65-51c4b101fb5d","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-11-03T15:50:11.887Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"276628b3-bba1-4e0b-9d65-51c4b101fb5d","group":"small-group","indices":{"0":true,"1":true}}
{"level":"info","ts":"2026-11-03T15:50:11.887Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"276628b3-bba1-4e0b-9d65-51c4b101fb5d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:50:11.887Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"276628b3-bba1-4e0b-9d65-51c4b101fb5d","podName":"ray-cluster-small-group-worker-xbgwc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:50:11.887Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"276628b3-bba1-4e0b-9d65-51c4b101fb5d","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:50:11.887Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"276628b3-bba1-4e0b-9d65-51c4b101fb5d","seconds":300}
{"level":"info","ts":"2026-11-03T15:50:14.950Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"39313a0e-fd11-40ac-becd-f2d31d4dfd34"}
{"level":"info","ts":"2026-11-03T15:50:14.950Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"39313a0e-fd11-40ac-becd-f2d31d4dfd34"}
{"level":"info","ts":"2026-11-03T15:50:14.950Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"39313a0e-fd11-40ac-becd-f2d31d4dfd34","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:50:14.950Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"39313a0e-fd11-40ac-becd-f2d31d4dfd34","Found 1 head Pod":"ray-cluster-head-jhq58","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:50:14.950Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"39313a0e-fd11-40ac-becd-f2d31d4dfd34","head Pod":"ray-cluster-head-jhq58","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-jhq58. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:50:14.950Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"39313a0e-fd11-40ac-becd-f2d31d4dfd34","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-11-03T15:50:14.950Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"39313a0e-fd11-40ac-becd-f2d31d4dfd34","worker Pod":"ray-cluster-small-group-worker-xbgwc","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-xbgwc. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:50:14.950Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"39313a0e-fd11-40ac-becd-f2d31d4dfd34","worker Pod":"ray-cluster-small-group-worker-lc796","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-lc796. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:50:14.950Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"39313a0e-fd11-40ac-becd-f2d31d4dfd34","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:50:14.950Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"39313a0e-fd11-40ac-becd-f2d31d4dfd34","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-11-03T15:50:14.950Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"39313a0e-fd11-40ac-becd-f2d31d4dfd34","group":"small-group","indices":{"0":true,"1":true}}
{"level":"info","ts":"2026-11-03T15:50:14.950Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"39313a0e-fd11-40ac-becd-f2d31d4dfd34","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:50:14.950Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"39313a0e-fd11-40ac-becd-f2d31d4dfd34","podName":"ray-cluster-small-group-worker-xbgwc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:50:14.950Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"39313a0e-fd11-40ac-becd-f2d31d4dfd34","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:50:14.950Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"39313a0e-fd11-40ac-becd-f2d31d4dfd34","seconds":300}
{"level":"info","ts":"2026-11-03T15:50:15.906Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4a248643-5307-4dca-abf4-0610f04d058b"}
{"level":"info","ts":"2026-11-03T15:50:15.906Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4a248643-5307-4dca-abf4-0610f04d058b"}
{"level":"info","ts":"2026-11-03T15:50:15.906Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4a248643-5307-4dca-abf4-0610f04d058b","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:50:15.906Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4a248643-5307-4dca-abf4-0610f04d058b","Found 1 head Pod":"ray-cluster-head-jhq58","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:50:15.906Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4a248643-5307-4dca-abf4-0610f04d058b","head Pod":"ray-cluster-head-jhq58","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-jhq58. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:50:15.906Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4a248643-5307-4dca-abf4-0610f04d058b","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-11-03T15:50:15.906Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4a248643-5307-4dca-abf4-0610f04d058b","worker Pod":"ray-cluster-small-group-worker-xbgwc","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-xbgwc. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:50:15.906Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4a248643-5307-4dca-abf4-0610f04d058b","worker Pod":"ray-cluster-small-group-worker-lc796","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-lc796. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:50:15.906Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4a248643-5307-4dca-abf4-0610f04d058b","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:50:15.906Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4a248643-5307-4dca-abf4-0610f04d058b","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-11-03T15:50:15.906Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4a248643-5307-4dca-abf4-0610f04d058b","group":"small-group","indices":{"0":true,"1":true}}
{"level":"info","ts":"2026-11-03T15:50:15.906Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4a248643-5307-4dca-abf4-0610f04d058b","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:50:15.907Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4a248643-5307-4dca-abf4-0610f04d058b","podName":"ray-cluster-small-group-worker-lc796","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:50:15.907Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4a248643-5307-4dca-abf4-0610f04d058b","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:50:15.907Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4a248643-5307-4dca-abf4-0610f04d058b","seconds":300}
{"level":"info","ts":"2026-11-03T15:50:20.978Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"ca212312-cdb6-40a2-ad83-71211818c269"}
{"level":"info","ts":"2026-11-03T15:50:20.978Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"ca212312-cdb6-40a2-ad83-71211818c269"}
{"level":"info","ts":"2026-11-03T15:50:20.978Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"ca212312-cdb6-40a2-ad83-71211818c269","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:50:20.979Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"ca212312-cdb6-40a2-ad83-71211818c269","Found 1 head Pod":"ray-cluster-head-jhq58","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:50:20.979Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"ca212312-cdb6-40a2-ad83-71211818c269","head Pod":"ray-cluster-head-jhq58","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-jhq58. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:50:20.979Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"ca212312-cdb6-40a2-ad83-71211818c269","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-11-03T15:50:20.979Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"ca212312-cdb6-40a2-ad83-71211818c269","worker Pod":"ray-cluster-small-group-worker-xbgwc","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-xbgwc. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:50:20.979Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"ca212312-cdb6-40a2-ad83-71211818c269","worker Pod":"ray-cluster-small-group-worker-lc796","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-lc796. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:50:20.979Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"ca212312-cdb6-40a2-ad83-71211818c269","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:50:20.979Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"ca212312-cdb6-40a2-ad83-71211818c269","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-11-03T15:50:20.979Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"ca212312-cdb6-40a2-ad83-71211818c269","group":"small-group","indices":{"0":true,"1":true}}
{"level":"info","ts":"2026-11-03T15:50:20.979Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"ca212312-cdb6-40a2-ad83-71211818c269","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:50:20.979Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"ca212312-cdb6-40a2-ad83-71211818c269","podName":"ray-cluster-small-group-worker-lc796","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:50:20.979Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"ca212312-cdb6-40a2-ad83-71211818c269","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:50:20.979Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"ca212312-cdb6-40a2-ad83-71211818c269","seconds":300}
{"level":"info","ts":"2026-11-03T15:50:21.985Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"27cce15c-8db9-472f-b396-626d908362fe"}
{"level":"info","ts":"2026-11-03T15:50:21.985Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"27cce15c-8db9-472f-b396-626d908362fe"}
{"level":"info","ts":"2026-11-03T15:50:21.985Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"27cce15c-8db9-472f-b396-626d908362fe","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:50:21.985Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"27cce15c-8db9-472f-b396-626d908362fe","Found 1 head Pod":"ray-cluster-head-jhq58","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:50:21.985Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"27cce15c-8db9-472f-b396-626d908362fe","head Pod":"ray-cluster-head-jhq58","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-jhq58. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:50:21.985Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"27cce15c-8db9-472f-b396-626d908362fe","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-11-03T15:50:21.985Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"27cce15c-8db9-472f-b396-626d908362fe","worker Pod":"ray-cluster-small-group-worker-xbgwc","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-xbgwc. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:50:21.985Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"27cce15c-8db9-472f-b396-626d908362fe","worker Pod":"ray-cluster-small-group-worker-lc796","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-lc796. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:50:21.985Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"27cce15c-8db9-472f-b396-626d908362fe","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:50:21.985Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"27cce15c-8db9-472f-b396-626d908362fe","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-11-03T15:50:21.985Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"27cce15c-8db9-472f-b396-626d908362fe","group":"small-group","indices":{"0":true,"1":true}}
{"level":"info","ts":"2026-11-03T15:50:21.985Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"27cce15c-8db9-472f-b396-626d908362fe","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:50:21.985Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"27cce15c-8db9-472f-b396-626d908362fe","podName":"ray-cluster-small-group-worker-lc796","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:48:59Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:50:21.985Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"27cce15c-8db9-472f-b396-626d908362fe","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:48:59Z","stateTransitionTimes":{"ready":"2026-11-03T15:48:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"ray-cluster-head-jhq58","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-11-03T15:46:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:48:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:48:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3},"new status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:50:21Z","stateTransitionTimes":{"ready":"2026-11-03T15:48:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"ray-cluster-head-jhq58","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-11-03T15:46:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:48:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:48:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspending","message":""}],"availableWorkerReplicas":1,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3}}
{"level":"info","ts":"2026-11-03T15:50:22.047Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"94f04e4b-5992-4876-b503-10e44b9be90e","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:50:24.048Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4318e2c4-3c4e-4594-9bca-085706661262"}
{"level":"info","ts":"2026-11-03T15:50:24.048Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4318e2c4-3c4e-4594-9bca-085706661262"}
{"level":"info","ts":"2026-11-03T15:50:24.048Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4318e2c4-3c4e-4594-9bca-085706661262","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:50:24.048Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4318e2c4-3c4e-4594-9bca-085706661262","Found 1 head Pod":"ray-cluster-head-jhq58","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:50:24.048Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4318e2c4-3c4e-4594-9bca-085706661262","head Pod":"ray-cluster-head-jhq58","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-jhq58. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:50:24.048Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4318e2c4-3c4e-4594-9bca-085706661262","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-11-03T15:50:24.048Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4318e2c4-3c4e-4594-9bca-085706661262","worker Pod":"ray-cluster-small-group-worker-xbgwc","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-xbgwc. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:50:24.048Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4318e2c4-3c4e-4594-9bca-085706661262","worker Pod":"ray-cluster-small-group-worker-lc796","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-lc796. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:50:24.048Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4318e2c4-3c4e-4594-9bca-085706661262","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:50:24.048Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4318e2c4-3c4e-4594-9bca-085706661262","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-11-03T15:50:24.048Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4318e2c4-3c4e-4594-9bca-085706661262","group":"small-group","indices":{"0":true,"1":true}}
{"level":"info","ts":"2026-11-03T15:50:24.048Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4318e2c4-3c4e-4594-9bca-085706661262","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:50:24.048Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4318e2c4-3c4e-4594-9bca-085706661262","podName":"ray-cluster-small-group-worker-xbgwc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:50:24.048Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4318e2c4-3c4e-4594-9bca-085706661262","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:50:24.048Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4318e2c4-3c4e-4594-9bca-085706661262","seconds":300}
{"level":"info","ts":"2026-11-03T15:50:24.998Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"6c18c997-c6b9-4e90-884e-e5d9c533cba4"}
{"level":"info","ts":"2026-11-03T15:50:24.998Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"6c18c997-c6b9-4e90-884e-e5d9c533cba4"}
{"level":"info","ts":"2026-11-03T15:50:24.999Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"6c18c997-c6b9-4e90-884e-e5d9c533cba4","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:50:24.999Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"6c18c997-c6b9-4e90-884e-e5d9c533cba4","Found 1 head Pod":"ray-cluster-head-jhq58","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:50:24.999Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"6c18c997-c6b9-4e90-884e-e5d9c533cba4","head Pod":"ray-cluster-head-jhq58","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-jhq58. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:50:24.999Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"6c18c997-c6b9-4e90-884e-e5d9c533cba4","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-11-03T15:50:24.999Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"6c18c997-c6b9-4e90-884e-e5d9c533cba4","worker Pod":"ray-cluster-small-group-worker-xbgwc","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-xbgwc. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:50:24.999Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"6c18c997-c6b9-4e90-884e-e5d9c533cba4","worker Pod":"ray-cluster-small-group-worker-lc796","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-lc796. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:50:24.999Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"6c18c997-c6b9-4e90-884e-e5d9c533cba4","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:50:24.999Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"6c18c997-c6b9-4e90-884e-e5d9c533cba4","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-11-03T15:50:24.999Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"6c18c997-c6b9-4e90-884e-e5d9c533cba4","group":"small-group","indices":{"0":true,"1":true}}
{"level":"info","ts":"2026-11-03T15:50:24.999Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"6c18c997-c6b9-4e90-884e-e5d9c533cba4","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:50:24.999Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"6c18c997-c6b9-4e90-884e-e5d9c533cba4","podName":"ray-cluster-small-group-worker-xbgwc","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:50:24.999Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"6c18c997-c6b9-4e90-884e-e5d9c533cba4","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:50:24.999Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"6c18c997-c6b9-4e90-884e-e5d9c533cba4","seconds":300}
{"level":"info","ts":"2026-11-03T15:50:25.954Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4c24c2ca-a0ea-4360-888b-dc72979cccee"}
{"level":"info","ts":"2026-11-03T15:50:25.954Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4c24c2ca-a0ea-4360-888b-dc72979cccee"}
{"level":"info","ts":"2026-11-03T15:50:25.954Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4c24c2ca-a0ea-4360-888b-dc72979cccee","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:50:25.954Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4c24c2ca-a0ea-4360-888b-dc72979cccee","Found 1 head Pod":"ray-cluster-head-jhq58","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:50:25.954Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4c24c2ca-a0ea-4360-888b-dc72979cccee","head Pod":"ray-cluster-head-jhq58","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-jhq58. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:50:25.954Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4c24c2ca-a0ea-4360-888b-dc72979cccee","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-11-03T15:50:25.954Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4c24c2ca-a0ea-4360-888b-dc72979cccee","worker Pod":"ray-cluster-small-group-worker-xbgwc","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-xbgwc. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:50:25.954Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4c24c2ca-a0ea-4360-888b-dc72979cccee","worker Pod":"ray-cluster-small-group-worker-lc796","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-lc796. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:50:25.954Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4c24c2ca-a0ea-4360-888b-dc72979cccee","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:50:25.954Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4c24c2ca-a0ea-4360-888b-dc72979cccee","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-11-03T15:50:25.954Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4c24c2ca-a0ea-4360-888b-dc72979cccee","group":"small-group","indices":{"0":true,"1":true}}
{"level":"info","ts":"2026-11-03T15:50:25.954Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4c24c2ca-a0ea-4360-888b-dc72979cccee","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:50:25.954Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4c24c2ca-a0ea-4360-888b-dc72979cccee","podName":"ray-cluster-small-group-worker-xbgwc","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:48:52Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:50:25.955Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4c24c2ca-a0ea-4360-888b-dc72979cccee","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:50:21Z","stateTransitionTimes":{"ready":"2026-11-03T15:48:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"ray-cluster-head-jhq58","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-11-03T15:46:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:48:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:48:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspending","message":""}],"availableWorkerReplicas":1,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3},"new status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:50:25Z","stateTransitionTimes":{"ready":"2026-11-03T15:48:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"ray-cluster-head-jhq58","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-11-03T15:46:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:48:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:48:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspending","message":""}],"availableWorkerReplicas":2,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3}}
{"level":"info","ts":"2026-11-03T15:50:26.016Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"edbc3116-7cda-40a3-95e5-9de640de619c","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:50:28.016Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"f977898a-fcc7-4284-b6d5-cb0dc525080f"}
{"level":"info","ts":"2026-11-03T15:50:28.016Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"f977898a-fcc7-4284-b6d5-cb0dc525080f"}
{"level":"info","ts":"2026-11-03T15:50:28.016Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"f977898a-fcc7-4284-b6d5-cb0dc525080f","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:50:28.016Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"f977898a-fcc7-4284-b6d5-cb0dc525080f","Found 1 head Pod":"ray-cluster-head-jhq58","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:50:28.016Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"f977898a-fcc7-4284-b6d5-cb0dc525080f","head Pod":"ray-cluster-head-jhq58","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-jhq58. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:50:28.016Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"f977898a-fcc7-4284-b6d5-cb0dc525080f","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-11-03T15:50:28.016Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"f977898a-fcc7-4284-b6d5-cb0dc525080f","worker Pod":"ray-cluster-small-group-worker-xbgwc","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-xbgwc. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:50:28.016Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"f977898a-fcc7-4284-b6d5-cb0dc525080f","worker Pod":"ray-cluster-small-group-worker-lc796","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-lc796. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:50:28.016Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"f977898a-fcc7-4284-b6d5-cb0dc525080f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:50:28.016Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"f977898a-fcc7-4284-b6d5-cb0dc525080f","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-11-03T15:50:28.016Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"f977898a-fcc7-4284-b6d5-cb0dc525080f","group":"small-group","indices":{"0":true,"1":true}}
{"level":"info","ts":"2026-11-03T15:50:28.016Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"f977898a-fcc7-4284-b6d5-cb0dc525080f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:50:28.017Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"f977898a-fcc7-4284-b6d5-cb0dc525080f","podName":"ray-cluster-small-group-worker-xbgwc","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:48:52Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:50:28.017Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"f977898a-fcc7-4284-b6d5-cb0dc525080f","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:50:28.017Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"f977898a-fcc7-4284-b6d5-cb0dc525080f","seconds":300}
{"level":"info","ts":"2026-11-03T15:50:32.994Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"e1624482-7381-462c-911d-02703414a516"}
{"level":"info","ts":"2026-11-03T15:50:32.994Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"e1624482-7381-462c-911d-02703414a516"}
{"level":"info","ts":"2026-11-03T15:50:32.994Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"e1624482-7381-462c-911d-02703414a516","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:50:32.994Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"e1624482-7381-462c-911d-02703414a516","Found 1 head Pod":"ray-cluster-head-jhq58","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:50:32.994Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"e1624482-7381-462c-911d-02703414a516","head Pod":"ray-cluster-head-jhq58","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-jhq58. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:50:32.994Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"e1624482-7381-462c-911d-02703414a516","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-11-03T15:50:32.994Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"e1624482-7381-462c-911d-02703414a516","worker Pod":"ray-cluster-small-group-worker-xbgwc","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-xbgwc. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:50:32.994Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"e1624482-7381-462c-911d-02703414a516","worker Pod":"ray-cluster-small-group-worker-lc796","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-lc796. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:50:32.994Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"e1624482-7381-462c-911d-02703414a516","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:50:32.994Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"e1624482-7381-462c-911d-02703414a516","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-11-03T15:50:32.994Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"e1624482-7381-462c-911d-02703414a516","group":"small-group","indices":{"0":true,"1":true}}
{"level":"info","ts":"2026-11-03T15:50:32.994Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"e1624482-7381-462c-911d-02703414a516","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:50:32.994Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"e1624482-7381-462c-911d-02703414a516","podName":"ray-cluster-small-group-worker-xbgwc","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:48:52Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:50:32.994Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"e1624482-7381-462c-911d-02703414a516","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:50:25Z","stateTransitionTimes":{"ready":"2026-11-03T15:48:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"ray-cluster-head-jhq58","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-11-03T15:46:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:48:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:48:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspending","message":""}],"availableWorkerReplicas":2,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3},"new status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:50:32Z","stateTransitionTimes":{"ready":"2026-11-03T15:48:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"ray-cluster-head-jhq58","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-11-03T15:46:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:48:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:48:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":2,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3}}
{"level":"info","ts":"2026-11-03T15:50:33.056Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"a06464a4-141a-4e27-86e2-467a6a9ed06a","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:50:35.056Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4fb8dcdb-d151-43b8-b248-7048ce933784"}
{"level":"info","ts":"2026-11-03T15:50:35.056Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4fb8dcdb-d151-43b8-b248-7048ce933784"}
{"level":"info","ts":"2026-11-03T15:50:35.056Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4fb8dcdb-d151-43b8-b248-7048ce933784","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:50:35.056Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4fb8dcdb-d151-43b8-b248-7048ce933784","Found 1 head Pod":"ray-cluster-head-jhq58","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:50:35.056Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4fb8dcdb-d151-43b8-b248-7048ce933784","head Pod":"ray-cluster-head-jhq58","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-jhq58. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:50:35.056Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4fb8dcdb-d151-43b8-b248-7048ce933784","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-11-03T15:50:35.056Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4fb8dcdb-d151-43b8-b248-7048ce933784","worker Pod":"ray-cluster-small-group-worker-xbgwc","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-xbgwc. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:50:35.056Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4fb8dcdb-d151-43b8-b248-7048ce933784","worker Pod":"ray-cluster-small-group-worker-lc796","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-lc796. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:50:35.056Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4fb8dcdb-d151-43b8-b248-7048ce933784","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:50:35.056Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4fb8dcdb-d151-43b8-b248-7048ce933784","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-11-03T15:50:35.056Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4fb8dcdb-d151-43b8-b248-7048ce933784","group":"small-group","indices":{"0":true,"1":true}}
{"level":"info","ts":"2026-11-03T15:50:35.056Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4fb8dcdb-d151-43b8-b248-7048ce933784","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:50:35.056Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4fb8dcdb-d151-43b8-b248-7048ce933784","podName":"ray-cluster-small-group-worker-xbgwc","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:48:52Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:50:35.056Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4fb8dcdb-d151-43b8-b248-7048ce933784","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:50:35.056Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"4fb8dcdb-d151-43b8-b248-7048ce933784","seconds":300}
{"level":"info","ts":"2026-11-03T15:50:37.018Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9b9ea60a-1ee2-42f4-9196-e1c8c1b67b0e"}
{"level":"info","ts":"2026-11-03T15:50:37.018Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9b9ea60a-1ee2-42f4-9196-e1c8c1b67b0e"}
{"level":"info","ts":"2026-11-03T15:50:37.018Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9b9ea60a-1ee2-42f4-9196-e1c8c1b67b0e","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:50:37.018Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9b9ea60a-1ee2-42f4-9196-e1c8c1b67b0e","Found 1 head Pod":"ray-cluster-head-jhq58","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:50:37.018Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9b9ea60a-1ee2-42f4-9196-e1c8c1b67b0e","head Pod":"ray-cluster-head-jhq58","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-jhq58. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:50:37.018Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9b9ea60a-1ee2-42f4-9196-e1c8c1b67b0e","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-11-03T15:50:37.018Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9b9ea60a-1ee2-42f4-9196-e1c8c1b67b0e","worker Pod":"ray-cluster-small-group-worker-xbgwc","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-xbgwc. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:50:37.018Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9b9ea60a-1ee2-42f4-9196-e1c8c1b67b0e","worker Pod":"ray-cluster-small-group-worker-lc796","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-lc796. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:50:37.018Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9b9ea60a-1ee2-42f4-9196-e1c8c1b67b0e","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:50:37.018Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9b9ea60a-1ee2-42f4-9196-e1c8c1b67b0e","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-11-03T15:50:37.018Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9b9ea60a-1ee2-42f4-9196-e1c8c1b67b0e","group":"small-group","indices":{"0":true,"1":true}}
{"level":"info","ts":"2026-11-03T15:50:37.018Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9b9ea60a-1ee2-42f4-9196-e1c8c1b67b0e","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:50:37.018Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9b9ea60a-1ee2-42f4-9196-e1c8c1b67b0e","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:50:32Z","stateTransitionTimes":{"ready":"2026-11-03T15:48:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"ray-cluster-head-jhq58","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-11-03T15:46:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:48:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:48:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":2,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3},"new status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:50:37Z","stateTransitionTimes":{"ready":"2026-11-03T15:48:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"ray-cluster-head-jhq58","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-11-03T15:46:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:48:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:48:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":2,"availableWorkerReplicas":2,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3}}
{"level":"info","ts":"2026-11-03T15:50:37.080Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"1997e10b-0d98-4b76-9a6c-f7468e12c3ee","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:50:39.081Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"ab07155f-3610-4ac0-9281-fb32ac951fda"}
{"level":"info","ts":"2026-11-03T15:50:39.081Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"ab07155f-3610-4ac0-9281-fb32ac951fda"}
{"level":"info","ts":"2026-11-03T15:50:39.081Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"ab07155f-3610-4ac0-9281-fb32ac951fda","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:50:39.081Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"ab07155f-3610-4ac0-9281-fb32ac951fda","Found 1 head Pod":"ray-cluster-head-jhq58","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:50:39.081Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"ab07155f-3610-4ac0-9281-fb32ac951fda","head Pod":"ray-cluster-head-jhq58","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-jhq58. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:50:39.081Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"ab07155f-3610-4ac0-9281-fb32ac951fda","desired workerReplicas (always adhering to minReplicas/maxReplica)":2,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":2}
{"level":"info","ts":"2026-11-03T15:50:39.081Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"ab07155f-3610-4ac0-9281-fb32ac951fda","worker Pod":"ray-cluster-small-group-worker-xbgwc","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-xbgwc. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:50:39.081Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"ab07155f-3610-4ac0-9281-fb32ac951fda","worker Pod":"ray-cluster-small-group-worker-lc796","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-lc796. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:50:39.081Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"ab07155f-3610-4ac0-9281-fb32ac951fda","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:50:39.081Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"ab07155f-3610-4ac0-9281-fb32ac951fda","workerReplicas":2,"NumOfHosts":1,"runningPods":2,"diff":0}
{"level":"info","ts":"2026-11-03T15:50:39.081Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"ab07155f-3610-4ac0-9281-fb32ac951fda","group":"small-group","indices":{"0":true,"1":true}}
{"level":"info","ts":"2026-11-03T15:50:39.081Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"ab07155f-3610-4ac0-9281-fb32ac951fda","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:50:39.082Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"ab07155f-3610-4ac0-9281-fb32ac951fda","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:50:39.082Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"ab07155f-3610-4ac0-9281-fb32ac951fda","seconds":300}
{"level":"info","ts":"2026-11-03T15:51:36.388Z","logger":"raycluster-resource","msg":"default","name":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:51:36.388Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"ray-cluster","namespace":"test-ns-zjpgs"}
{"level":"info","ts":"2026-11-03T15:51:36.564Z","logger":"raycluster-resource","msg":"validate update","name":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:51:36.620Z","logger":"controllers.NetworkPolicy","msg":"Reconciling NetworkPolicies for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"13faa865-af9c-4651-bde1-fb949ab47923","cluster":"ray-cluster","namespace":"test-ns-zjpgs"}
{"level":"info","ts":"2026-11-03T15:51:36.620Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"fa10738c-1c0c-4b1a-b24e-d255fdabfd96"}
{"level":"info","ts":"2026-11-03T15:51:36.620Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"fa10738c-1c0c-4b1a-b24e-d255fdabfd96"}
{"level":"info","ts":"2026-11-03T15:51:36.620Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"fa10738c-1c0c-4b1a-b24e-d255fdabfd96","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:51:36.621Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"884440ec-63c4-4379-a19c-a6dc9d44898a","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:51:36.621Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"fa10738c-1c0c-4b1a-b24e-d255fdabfd96","Found 1 head Pod":"ray-cluster-head-jhq58","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:51:36.621Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"fa10738c-1c0c-4b1a-b24e-d255fdabfd96","head Pod":"ray-cluster-head-jhq58","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-jhq58. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:51:36.621Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"fa10738c-1c0c-4b1a-b24e-d255fdabfd96","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":1}
{"level":"info","ts":"2026-11-03T15:51:36.621Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"fa10738c-1c0c-4b1a-b24e-d255fdabfd96","worker Pod":"ray-cluster-small-group-worker-xbgwc","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-xbgwc. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:51:36.621Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"fa10738c-1c0c-4b1a-b24e-d255fdabfd96","worker Pod":"ray-cluster-small-group-worker-lc796","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-lc796. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:51:36.621Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"fa10738c-1c0c-4b1a-b24e-d255fdabfd96","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:51:36.621Z","logger":"controllers.RayCluster","msg":"Deleting pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"fa10738c-1c0c-4b1a-b24e-d255fdabfd96","namespace":"test-ns-zjpgs","name":"ray-cluster-small-group-worker-xbgwc"}
{"level":"info","ts":"2026-11-03T15:51:36.621Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"namespace":"test-ns-zjpgs","name":"ray-cluster","reconcileID":"7490427c-7b46-4046-a963-925bca6dadf7","namespacedName":{"name":"ray-cluster","namespace":"test-ns-zjpgs"}}
{"level":"info","ts":"2026-11-03T15:51:36.621Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"namespace":"test-ns-zjpgs","name":"ray-cluster","reconcileID":"7490427c-7b46-4046-a963-925bca6dadf7","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:51:36.621Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"namespace":"test-ns-zjpgs","name":"ray-cluster","reconcileID":"7490427c-7b46-4046-a963-925bca6dadf7","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:51:36.719Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"fa10738c-1c0c-4b1a-b24e-d255fdabfd96","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:51:36.719Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"fa10738c-1c0c-4b1a-b24e-d255fdabfd96","group":"small-group","indices":{"1":true}}
{"level":"info","ts":"2026-11-03T15:51:36.719Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"fa10738c-1c0c-4b1a-b24e-d255fdabfd96","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:51:36.719Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"fa10738c-1c0c-4b1a-b24e-d255fdabfd96","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:50:37Z","stateTransitionTimes":{"ready":"2026-11-03T15:48:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"ray-cluster-head-jhq58","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-11-03T15:46:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:48:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:48:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":2,"availableWorkerReplicas":2,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3},"new status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:51:36Z","stateTransitionTimes":{"ready":"2026-11-03T15:48:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"ray-cluster-head-jhq58","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":3,"lastTransitionTime":"2026-11-03T15:46:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:48:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:48:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":2,"availableWorkerReplicas":2,"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":4}}
{"level":"info","ts":"2026-11-03T15:51:36.767Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"namespace":"test-ns-zjpgs","name":"ray-cluster","reconcileID":"7490427c-7b46-4046-a963-925bca6dadf7","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:51:36.776Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"fa10738c-1c0c-4b1a-b24e-d255fdabfd96","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-zjpgs","name":"ray-cluster"}}
{"level":"info","ts":"2026-11-03T15:51:36.776Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"fa10738c-1c0c-4b1a-b24e-d255fdabfd96"}
{"level":"error","ts":"2026-11-03T15:51:36.776Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"fa10738c-1c0c-4b1a-b24e-d255fdabfd96","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:51:36.776Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"bf8179b4-1941-4686-ad37-ebd0943d71cd"}
{"level":"info","ts":"2026-11-03T15:51:36.776Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"bf8179b4-1941-4686-ad37-ebd0943d71cd"}
{"level":"info","ts":"2026-11-03T15:51:36.776Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"bf8179b4-1941-4686-ad37-ebd0943d71cd","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:51:36.776Z","logger":"controllers.NetworkPolicy","msg":"Successfully reconciled NetworkPolicies for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"13faa865-af9c-4651-bde1-fb949ab47923","cluster":"ray-cluster","namespace":"test-ns-zjpgs"}
{"level":"info","ts":"2026-11-03T15:51:36.776Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"bf8179b4-1941-4686-ad37-ebd0943d71cd","Found 1 head Pod":"ray-cluster-head-jhq58","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:51:36.776Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"bf8179b4-1941-4686-ad37-ebd0943d71cd","head Pod":"ray-cluster-head-jhq58","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-jhq58. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:51:36.776Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"bf8179b4-1941-4686-ad37-ebd0943d71cd","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-11-03T15:51:36.776Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"bf8179b4-1941-4686-ad37-ebd0943d71cd","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:50:37Z","stateTransitionTimes":{"ready":"2026-11-03T15:48:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"ray-cluster-head-jhq58","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":4,"lastTransitionTime":"2026-11-03T15:46:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:48:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:48:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":2,"availableWorkerReplicas":2,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3},"new status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:51:36Z","stateTransitionTimes":{"ready":"2026-11-03T15:48:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"ray-cluster-head-jhq58","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":4,"lastTransitionTime":"2026-11-03T15:46:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:48:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:48:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":2,"availableWorkerReplicas":2,"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":4}}
{"level":"info","ts":"2026-11-03T15:51:36.837Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"47667ed9-e616-4dc7-b0e2-34d01082abd5"}
{"level":"info","ts":"2026-11-03T15:51:36.837Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"47667ed9-e616-4dc7-b0e2-34d01082abd5"}
{"level":"info","ts":"2026-11-03T15:51:36.837Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"47667ed9-e616-4dc7-b0e2-34d01082abd5","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:51:36.837Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"47667ed9-e616-4dc7-b0e2-34d01082abd5","Found 1 head Pod":"ray-cluster-head-jhq58","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:51:36.837Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"47667ed9-e616-4dc7-b0e2-34d01082abd5","head Pod":"ray-cluster-head-jhq58","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-jhq58. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:51:36.837Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"47667ed9-e616-4dc7-b0e2-34d01082abd5","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-11-03T15:51:36.838Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"47667ed9-e616-4dc7-b0e2-34d01082abd5","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"3","desiredMemory":"4G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:50:37Z","stateTransitionTimes":{"ready":"2026-11-03T15:48:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"ray-cluster-head-jhq58","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":4,"lastTransitionTime":"2026-11-03T15:46:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:48:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:48:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":2,"availableWorkerReplicas":2,"desiredWorkerReplicas":2,"maxWorkerReplicas":3,"observedGeneration":3},"new status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:51:36Z","stateTransitionTimes":{"ready":"2026-11-03T15:48:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"ray-cluster-head-jhq58","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":4,"lastTransitionTime":"2026-11-03T15:46:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:48:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:48:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":2,"availableWorkerReplicas":2,"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":4}}
{"level":"info","ts":"2026-11-03T15:51:36.893Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"47667ed9-e616-4dc7-b0e2-34d01082abd5","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-zjpgs","name":"ray-cluster"}}
{"level":"info","ts":"2026-11-03T15:51:36.893Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"47667ed9-e616-4dc7-b0e2-34d01082abd5"}
{"level":"error","ts":"2026-11-03T15:51:36.893Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"47667ed9-e616-4dc7-b0e2-34d01082abd5","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:51:36.898Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"3622320b-5ce5-480a-a354-9dbdadb09313"}
{"level":"info","ts":"2026-11-03T15:51:36.898Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"3622320b-5ce5-480a-a354-9dbdadb09313"}
{"level":"info","ts":"2026-11-03T15:51:36.898Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"3622320b-5ce5-480a-a354-9dbdadb09313","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:51:36.898Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"3622320b-5ce5-480a-a354-9dbdadb09313","Found 1 head Pod":"ray-cluster-head-jhq58","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:51:36.898Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"3622320b-5ce5-480a-a354-9dbdadb09313","head Pod":"ray-cluster-head-jhq58","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-jhq58. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:51:36.898Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"3622320b-5ce5-480a-a354-9dbdadb09313","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-11-03T15:51:36.899Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"3622320b-5ce5-480a-a354-9dbdadb09313","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:51:36.899Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"3622320b-5ce5-480a-a354-9dbdadb09313","seconds":300}
{"level":"info","ts":"2026-11-03T15:51:37.081Z","logger":"controllers.RayClusterMTLS","msg":"Updating worker certificate","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"884440ec-63c4-4379-a19c-a6dc9d44898a","certificate":"ray-worker-cert-ray-cluster","ipAddresses":["10.134.0.58","127.0.0.1"]}
{"level":"info","ts":"2026-11-03T15:51:37.248Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"d0e4d764-7f2d-4c86-8f42-4734c083e8e3"}
{"level":"info","ts":"2026-11-03T15:51:37.248Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"d0e4d764-7f2d-4c86-8f42-4734c083e8e3"}
{"level":"info","ts":"2026-11-03T15:51:37.248Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"d0e4d764-7f2d-4c86-8f42-4734c083e8e3","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:51:37.248Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"d0e4d764-7f2d-4c86-8f42-4734c083e8e3","Found 1 head Pod":"ray-cluster-head-jhq58","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:51:37.248Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"d0e4d764-7f2d-4c86-8f42-4734c083e8e3","head Pod":"ray-cluster-head-jhq58","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-jhq58. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:51:37.248Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"d0e4d764-7f2d-4c86-8f42-4734c083e8e3","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-11-03T15:51:37.249Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"d0e4d764-7f2d-4c86-8f42-4734c083e8e3","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:51:36Z","stateTransitionTimes":{"ready":"2026-11-03T15:48:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"ray-cluster-head-jhq58","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":4,"lastTransitionTime":"2026-11-03T15:46:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:48:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:48:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":2,"availableWorkerReplicas":2,"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":4},"new status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:51:37Z","stateTransitionTimes":{"ready":"2026-11-03T15:48:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"ray-cluster-head-jhq58","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":4,"lastTransitionTime":"2026-11-03T15:46:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:48:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:48:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":4}}
{"level":"info","ts":"2026-11-03T15:51:37.517Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"884440ec-63c4-4379-a19c-a6dc9d44898a","certificate":"ray-worker-cert-ray-cluster"}
{"level":"info","ts":"2026-11-03T15:51:37.517Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"884440ec-63c4-4379-a19c-a6dc9d44898a"}
{"level":"info","ts":"2026-11-03T15:51:37.517Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"acdb204e-aa8e-40e1-8c93-e76d605e1903","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:51:38.172Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"acdb204e-aa8e-40e1-8c93-e76d605e1903","certificate":"ray-worker-cert-ray-cluster"}
{"level":"info","ts":"2026-11-03T15:51:38.172Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"acdb204e-aa8e-40e1-8c93-e76d605e1903"}
{"level":"info","ts":"2026-11-03T15:51:38.339Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"94494063-e4fa-4b98-b18c-f1eb55ed3cf1"}
{"level":"info","ts":"2026-11-03T15:51:38.339Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"94494063-e4fa-4b98-b18c-f1eb55ed3cf1"}
{"level":"info","ts":"2026-11-03T15:51:38.339Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"94494063-e4fa-4b98-b18c-f1eb55ed3cf1","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:51:38.339Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"94494063-e4fa-4b98-b18c-f1eb55ed3cf1","Found 1 head Pod":"ray-cluster-head-jhq58","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:51:38.339Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"94494063-e4fa-4b98-b18c-f1eb55ed3cf1","head Pod":"ray-cluster-head-jhq58","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-jhq58. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:51:38.339Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"94494063-e4fa-4b98-b18c-f1eb55ed3cf1","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-11-03T15:51:38.340Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"94494063-e4fa-4b98-b18c-f1eb55ed3cf1","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:51:38.340Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"94494063-e4fa-4b98-b18c-f1eb55ed3cf1","seconds":300}
{"level":"info","ts":"2026-11-03T15:51:38.343Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"3370c900-7806-4573-8f08-312b236391f3"}
{"level":"info","ts":"2026-11-03T15:51:38.344Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"3370c900-7806-4573-8f08-312b236391f3"}
{"level":"info","ts":"2026-11-03T15:51:38.344Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"3370c900-7806-4573-8f08-312b236391f3","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:51:38.344Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"3370c900-7806-4573-8f08-312b236391f3","Found 1 head Pod":"ray-cluster-head-jhq58","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:51:38.344Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"3370c900-7806-4573-8f08-312b236391f3","head Pod":"ray-cluster-head-jhq58","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-jhq58. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:51:38.344Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"3370c900-7806-4573-8f08-312b236391f3","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":1}
{"level":"info","ts":"2026-11-03T15:51:38.344Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"3370c900-7806-4573-8f08-312b236391f3","worker Pod":"ray-cluster-small-group-worker-lc796","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-lc796. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:51:38.344Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"3370c900-7806-4573-8f08-312b236391f3","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:51:38.344Z","logger":"controllers.RayCluster","msg":"Deleting pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"3370c900-7806-4573-8f08-312b236391f3","namespace":"test-ns-zjpgs","name":"ray-cluster-small-group-worker-xbgwc"}
{"level":"info","ts":"2026-11-03T15:51:38.393Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"3370c900-7806-4573-8f08-312b236391f3","The worker Pod has already been deleted":"ray-cluster-small-group-worker-xbgwc"}
{"level":"info","ts":"2026-11-03T15:51:38.393Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"3370c900-7806-4573-8f08-312b236391f3","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:51:38.393Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"3370c900-7806-4573-8f08-312b236391f3","group":"small-group","indices":{"1":true}}
{"level":"info","ts":"2026-11-03T15:51:38.393Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"3370c900-7806-4573-8f08-312b236391f3","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:51:38.394Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"3370c900-7806-4573-8f08-312b236391f3","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:51:38.394Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"3370c900-7806-4573-8f08-312b236391f3","seconds":300}
{"level":"info","ts":"2026-11-03T15:52:08.174Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"75af0f43-016e-458f-a3c5-c401ceb9b704","cluster":"ray-cluster"}
{"level":"error","ts":"2026-11-03T15:52:46.491Z","logger":"events","msg":"Server rejected event (will not retry!)","event":{"namespace":"test-ns-4jsqd","name":"ray-cluster.18e4934055e60516"},"error":"namespaces \"test-ns-4jsqd\" not found","stacktrace":"k8s.io/client-go/tools/events.recordEvent\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:270\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).finishSeries\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:153\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext.func1\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:255\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:256\nk8s.io/apimachinery/pkg/util/wait.JitterUntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:223\nk8s.io/apimachinery/pkg/util/wait.UntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:172"}
{"level":"error","ts":"2026-11-03T15:52:46.642Z","logger":"events","msg":"Server rejected event (will not retry!)","event":{"namespace":"test-ns-4jsqd","name":"ray-cluster.18e4935da3db3d01"},"error":"namespaces \"test-ns-4jsqd\" not found","stacktrace":"k8s.io/client-go/tools/events.recordEvent\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:270\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).finishSeries\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:153\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext.func1\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:255\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:256\nk8s.io/apimachinery/pkg/util/wait.JitterUntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:223\nk8s.io/apimachinery/pkg/util/wait.UntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.0/pkg/util/wait/backoff.go:172"}
{"level":"info","ts":"2026-11-03T15:52:46.874Z","logger":"raycluster-resource","msg":"default","name":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:52:46.874Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"ray-cluster","namespace":"test-ns-zjpgs"}
{"level":"info","ts":"2026-11-03T15:52:46.931Z","logger":"raycluster-resource","msg":"validate update","name":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:52:46.987Z","logger":"controllers.NetworkPolicy","msg":"Reconciling NetworkPolicies for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9f680761-e773-43a3-b5fa-235cd74b79eb","cluster":"ray-cluster","namespace":"test-ns-zjpgs"}
{"level":"info","ts":"2026-11-03T15:52:46.987Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"debe6381-fed3-47c4-9880-3c9e5bbec391","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:52:46.987Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"namespace":"test-ns-zjpgs","name":"ray-cluster","reconcileID":"68acfc2b-5bf2-4583-8e42-b21cfff76738","namespacedName":{"name":"ray-cluster","namespace":"test-ns-zjpgs"}}
{"level":"info","ts":"2026-11-03T15:52:46.987Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"namespace":"test-ns-zjpgs","name":"ray-cluster","reconcileID":"68acfc2b-5bf2-4583-8e42-b21cfff76738","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:52:46.987Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"namespace":"test-ns-zjpgs","name":"ray-cluster","reconcileID":"68acfc2b-5bf2-4583-8e42-b21cfff76738","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:52:46.987Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"2315f1ed-e4d6-40c0-b60f-ddd1e1a1158d"}
{"level":"info","ts":"2026-11-03T15:52:46.987Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"2315f1ed-e4d6-40c0-b60f-ddd1e1a1158d"}
{"level":"info","ts":"2026-11-03T15:52:46.987Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"2315f1ed-e4d6-40c0-b60f-ddd1e1a1158d","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:52:46.987Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"2315f1ed-e4d6-40c0-b60f-ddd1e1a1158d","Found 1 head Pod":"ray-cluster-head-jhq58","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:52:46.987Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"2315f1ed-e4d6-40c0-b60f-ddd1e1a1158d","head Pod":"ray-cluster-head-jhq58","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-jhq58. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:52:46.987Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"2315f1ed-e4d6-40c0-b60f-ddd1e1a1158d","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-11-03T15:52:46.988Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"2315f1ed-e4d6-40c0-b60f-ddd1e1a1158d","worker Pod":"ray-cluster-small-group-worker-lc796","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod ray-cluster-small-group-worker-lc796. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:52:46.988Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"2315f1ed-e4d6-40c0-b60f-ddd1e1a1158d","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:52:46.988Z","logger":"controllers.RayCluster","msg":"Deleting pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"2315f1ed-e4d6-40c0-b60f-ddd1e1a1158d","namespace":"test-ns-zjpgs","name":"ray-cluster-small-group-worker-lc796"}
{"level":"info","ts":"2026-11-03T15:52:47.086Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"namespace":"test-ns-zjpgs","name":"ray-cluster","reconcileID":"68acfc2b-5bf2-4583-8e42-b21cfff76738","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:52:47.099Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"2315f1ed-e4d6-40c0-b60f-ddd1e1a1158d","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-11-03T15:52:47.099Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"2315f1ed-e4d6-40c0-b60f-ddd1e1a1158d","group":"small-group","indices":{}}
{"level":"info","ts":"2026-11-03T15:52:47.099Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"2315f1ed-e4d6-40c0-b60f-ddd1e1a1158d","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:52:47.100Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"2315f1ed-e4d6-40c0-b60f-ddd1e1a1158d","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:51:37Z","stateTransitionTimes":{"ready":"2026-11-03T15:48:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"ray-cluster-head-jhq58","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":4,"lastTransitionTime":"2026-11-03T15:46:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:48:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:48:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":4},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:52:47Z","stateTransitionTimes":{"ready":"2026-11-03T15:48:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"ray-cluster-head-jhq58","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":4,"lastTransitionTime":"2026-11-03T15:46:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:48:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:48:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":5}}
{"level":"info","ts":"2026-11-03T15:52:47.142Z","logger":"controllers.NetworkPolicy","msg":"Successfully reconciled NetworkPolicies for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"9f680761-e773-43a3-b5fa-235cd74b79eb","cluster":"ray-cluster","namespace":"test-ns-zjpgs"}
{"level":"info","ts":"2026-11-03T15:52:47.154Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"2315f1ed-e4d6-40c0-b60f-ddd1e1a1158d","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-zjpgs","name":"ray-cluster"}}
{"level":"info","ts":"2026-11-03T15:52:47.154Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"2315f1ed-e4d6-40c0-b60f-ddd1e1a1158d"}
{"level":"error","ts":"2026-11-03T15:52:47.154Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"2315f1ed-e4d6-40c0-b60f-ddd1e1a1158d","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:52:47.155Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"23eb2945-db82-45c5-b014-d039a80b4dd9"}
{"level":"info","ts":"2026-11-03T15:52:47.155Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"23eb2945-db82-45c5-b014-d039a80b4dd9"}
{"level":"info","ts":"2026-11-03T15:52:47.155Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"23eb2945-db82-45c5-b014-d039a80b4dd9","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:52:47.155Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"23eb2945-db82-45c5-b014-d039a80b4dd9","Found 1 head Pod":"ray-cluster-head-jhq58","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:52:47.155Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"23eb2945-db82-45c5-b014-d039a80b4dd9","head Pod":"ray-cluster-head-jhq58","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-jhq58. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:52:47.155Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"23eb2945-db82-45c5-b014-d039a80b4dd9","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-11-03T15:52:47.155Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"23eb2945-db82-45c5-b014-d039a80b4dd9","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"2","desiredMemory":"3G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:51:37Z","stateTransitionTimes":{"ready":"2026-11-03T15:48:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"ray-cluster-head-jhq58","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-11-03T15:46:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:48:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:48:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":4},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:52:47Z","stateTransitionTimes":{"ready":"2026-11-03T15:48:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"ray-cluster-head-jhq58","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-11-03T15:46:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:48:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:48:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":5}}
{"level":"info","ts":"2026-11-03T15:52:47.229Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"30f0dfe7-d525-46db-922a-e4b1d121ba0e"}
{"level":"info","ts":"2026-11-03T15:52:47.229Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"30f0dfe7-d525-46db-922a-e4b1d121ba0e"}
{"level":"info","ts":"2026-11-03T15:52:47.229Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"30f0dfe7-d525-46db-922a-e4b1d121ba0e","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:52:47.229Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"30f0dfe7-d525-46db-922a-e4b1d121ba0e","Found 1 head Pod":"ray-cluster-head-jhq58","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:52:47.229Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"30f0dfe7-d525-46db-922a-e4b1d121ba0e","head Pod":"ray-cluster-head-jhq58","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-jhq58. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:52:47.229Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"30f0dfe7-d525-46db-922a-e4b1d121ba0e","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-11-03T15:52:47.230Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"30f0dfe7-d525-46db-922a-e4b1d121ba0e","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:52:47.230Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"30f0dfe7-d525-46db-922a-e4b1d121ba0e","seconds":300}
{"level":"info","ts":"2026-11-03T15:52:47.459Z","logger":"controllers.RayClusterMTLS","msg":"Updating worker certificate","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"debe6381-fed3-47c4-9880-3c9e5bbec391","certificate":"ray-worker-cert-ray-cluster","ipAddresses":["127.0.0.1"]}
{"level":"info","ts":"2026-11-03T15:52:47.664Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"1b956c89-a1e3-4f45-a80a-b4001f76d31a"}
{"level":"info","ts":"2026-11-03T15:52:47.664Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"1b956c89-a1e3-4f45-a80a-b4001f76d31a"}
{"level":"info","ts":"2026-11-03T15:52:47.664Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"1b956c89-a1e3-4f45-a80a-b4001f76d31a","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:52:47.664Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"1b956c89-a1e3-4f45-a80a-b4001f76d31a","Found 1 head Pod":"ray-cluster-head-jhq58","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:52:47.664Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"1b956c89-a1e3-4f45-a80a-b4001f76d31a","head Pod":"ray-cluster-head-jhq58","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-jhq58. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:52:47.664Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"1b956c89-a1e3-4f45-a80a-b4001f76d31a","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-11-03T15:52:47.664Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"1b956c89-a1e3-4f45-a80a-b4001f76d31a","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:52:47Z","stateTransitionTimes":{"ready":"2026-11-03T15:48:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"ray-cluster-head-jhq58","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-11-03T15:46:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:48:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:48:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspending","message":""}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"maxWorkerReplicas":3,"observedGeneration":5},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:52:47Z","stateTransitionTimes":{"ready":"2026-11-03T15:48:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"ray-cluster-head-jhq58","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-11-03T15:46:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:48:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:48:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":5}}
{"level":"info","ts":"2026-11-03T15:52:47.781Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"debe6381-fed3-47c4-9880-3c9e5bbec391","certificate":"ray-worker-cert-ray-cluster"}
{"level":"info","ts":"2026-11-03T15:52:47.781Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"debe6381-fed3-47c4-9880-3c9e5bbec391"}
{"level":"info","ts":"2026-11-03T15:52:47.781Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"36cbe1f7-b4fa-4942-9c91-79708fa3ed3c","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:52:48.437Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"36cbe1f7-b4fa-4942-9c91-79708fa3ed3c","certificate":"ray-worker-cert-ray-cluster"}
{"level":"info","ts":"2026-11-03T15:52:48.437Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"36cbe1f7-b4fa-4942-9c91-79708fa3ed3c"}
{"level":"info","ts":"2026-11-03T15:52:48.630Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"506875c2-a072-4088-ab4e-7e994d934807"}
{"level":"info","ts":"2026-11-03T15:52:48.630Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"506875c2-a072-4088-ab4e-7e994d934807"}
{"level":"info","ts":"2026-11-03T15:52:48.630Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"506875c2-a072-4088-ab4e-7e994d934807","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:52:48.630Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"506875c2-a072-4088-ab4e-7e994d934807","Found 1 head Pod":"ray-cluster-head-jhq58","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:52:48.630Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"506875c2-a072-4088-ab4e-7e994d934807","head Pod":"ray-cluster-head-jhq58","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-jhq58. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:52:48.630Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"506875c2-a072-4088-ab4e-7e994d934807","worker group":"small-group","Expectation":"NotSatisfiedGroupExpectations, reconcile the group later"}
{"level":"info","ts":"2026-11-03T15:52:48.630Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"506875c2-a072-4088-ab4e-7e994d934807","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:52:48.630Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"506875c2-a072-4088-ab4e-7e994d934807","seconds":300}
{"level":"info","ts":"2026-11-03T15:52:48.635Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"0edc7e36-16e6-4c7c-8236-ea70ae5d4541"}
{"level":"info","ts":"2026-11-03T15:52:48.635Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"0edc7e36-16e6-4c7c-8236-ea70ae5d4541"}
{"level":"info","ts":"2026-11-03T15:52:48.635Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"0edc7e36-16e6-4c7c-8236-ea70ae5d4541","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:52:48.636Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"0edc7e36-16e6-4c7c-8236-ea70ae5d4541","Found 1 head Pod":"ray-cluster-head-jhq58","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:52:48.636Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"0edc7e36-16e6-4c7c-8236-ea70ae5d4541","head Pod":"ray-cluster-head-jhq58","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-jhq58. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:52:48.636Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"0edc7e36-16e6-4c7c-8236-ea70ae5d4541","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-11-03T15:52:48.636Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"0edc7e36-16e6-4c7c-8236-ea70ae5d4541","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:52:48.636Z","logger":"controllers.RayCluster","msg":"Deleting pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"0edc7e36-16e6-4c7c-8236-ea70ae5d4541","namespace":"test-ns-zjpgs","name":"ray-cluster-small-group-worker-lc796"}
{"level":"info","ts":"2026-11-03T15:52:48.685Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"0edc7e36-16e6-4c7c-8236-ea70ae5d4541","The worker Pod has already been deleted":"ray-cluster-small-group-worker-lc796"}
{"level":"info","ts":"2026-11-03T15:52:48.685Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"0edc7e36-16e6-4c7c-8236-ea70ae5d4541","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-11-03T15:52:48.685Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"0edc7e36-16e6-4c7c-8236-ea70ae5d4541","group":"small-group","indices":{}}
{"level":"info","ts":"2026-11-03T15:52:48.685Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"0edc7e36-16e6-4c7c-8236-ea70ae5d4541","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:52:48.685Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"0edc7e36-16e6-4c7c-8236-ea70ae5d4541","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:52:48.685Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"0edc7e36-16e6-4c7c-8236-ea70ae5d4541","seconds":300}
{"level":"info","ts":"2026-11-03T15:52:51.104Z","logger":"raycluster-resource","msg":"default","name":"raycluster-auth-token"}
{"level":"info","ts":"2026-11-03T15:52:51.104Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"raycluster-auth-token","namespace":"test-ns-rhs44"}
{"level":"info","ts":"2026-11-03T15:52:51.160Z","logger":"raycluster-resource","msg":"validate create","name":"raycluster-auth-token"}
{"level":"info","ts":"2026-11-03T15:52:51.214Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"4b666730-ab1e-44db-a1ae-b6eaae0322d9","cluster":"raycluster-auth-token"}
{"level":"info","ts":"2026-11-03T15:52:51.214Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"namespace":"test-ns-rhs44","name":"raycluster-auth-token","reconcileID":"9ac2e95e-a308-4e13-8e92-b780ccb860a2","namespacedName":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"}}
{"level":"info","ts":"2026-11-03T15:52:51.214Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"namespace":"test-ns-rhs44","name":"raycluster-auth-token","reconcileID":"9ac2e95e-a308-4e13-8e92-b780ccb860a2","mode":"IntegratedOAuth","cluster":"raycluster-auth-token"}
{"level":"info","ts":"2026-11-03T15:52:51.214Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"namespace":"test-ns-rhs44","name":"raycluster-auth-token","reconcileID":"9ac2e95e-a308-4e13-8e92-b780ccb860a2","cluster":"raycluster-auth-token"}
{"level":"info","ts":"2026-11-03T15:52:51.214Z","logger":"authentication-controller","msg":"Adding authentication finalizer to RayCluster","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"namespace":"test-ns-rhs44","name":"raycluster-auth-token","reconcileID":"9ac2e95e-a308-4e13-8e92-b780ccb860a2","cluster":"raycluster-auth-token"}
{"level":"info","ts":"2026-11-03T15:52:51.214Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"4f935402-4623-4a9f-8665-daa71408a900"}
{"level":"info","ts":"2026-11-03T15:52:51.214Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"4f935402-4623-4a9f-8665-daa71408a900"}
{"level":"info","ts":"2026-11-03T15:52:51.214Z","logger":"controllers.NetworkPolicy","msg":"Reconciling NetworkPolicies for RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"0520d6ad-0f74-4323-8f4a-b68880bcf1cf","cluster":"raycluster-auth-token","namespace":"test-ns-rhs44"}
{"level":"info","ts":"2026-11-03T15:52:51.269Z","logger":"controllers.NetworkPolicy","msg":"Successfully created NetworkPolicy","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"0520d6ad-0f74-4323-8f4a-b68880bcf1cf","name":"raycluster-auth-token-head"}
{"level":"info","ts":"2026-11-03T15:52:51.271Z","logger":"raycluster-resource","msg":"default","name":"raycluster-auth-token"}
{"level":"info","ts":"2026-11-03T15:52:51.271Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"raycluster-auth-token","namespace":"test-ns-rhs44"}
{"level":"info","ts":"2026-11-03T15:52:51.322Z","logger":"controllers.RayCluster","msg":"Created service for RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"4f935402-4623-4a9f-8665-daa71408a900","name":"raycluster-auth-token-head-svc"}
{"level":"info","ts":"2026-11-03T15:52:51.322Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"4f935402-4623-4a9f-8665-daa71408a900"}
{"level":"info","ts":"2026-11-03T15:52:51.323Z","logger":"controllers.NetworkPolicy","msg":"Successfully created NetworkPolicy","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"0520d6ad-0f74-4323-8f4a-b68880bcf1cf","name":"raycluster-auth-token-workers-small-group"}
{"level":"info","ts":"2026-11-03T15:52:51.323Z","logger":"controllers.NetworkPolicy","msg":"Successfully reconciled NetworkPolicies for RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"0520d6ad-0f74-4323-8f4a-b68880bcf1cf","cluster":"raycluster-auth-token","namespace":"test-ns-rhs44"}
{"level":"info","ts":"2026-11-03T15:52:51.323Z","logger":"controllers.NetworkPolicy","msg":"Reconciling NetworkPolicies for RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"bc082e18-39b4-4a63-b506-bfc997d0a87b","cluster":"raycluster-auth-token","namespace":"test-ns-rhs44"}
{"level":"info","ts":"2026-11-03T15:52:51.325Z","logger":"raycluster-resource","msg":"validate update","name":"raycluster-auth-token"}
{"level":"info","ts":"2026-11-03T15:52:51.371Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"4f935402-4623-4a9f-8665-daa71408a900","error":"certificate ray-head-cert-raycluster-auth-token not found: certificates.cert-manager.io \"ray-head-cert-raycluster-auth-token\" not found"}
{"level":"error","ts":"2026-11-03T15:52:51.371Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"4f935402-4623-4a9f-8665-daa71408a900","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster-auth-token not found: certificates.cert-manager.io \"ray-head-cert-raycluster-auth-token\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:52:51.371Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"4f935402-4623-4a9f-8665-daa71408a900","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster-auth-token not found: certificates.cert-manager.io \"ray-head-cert-raycluster-auth-token\" not found"}
{"level":"info","ts":"2026-11-03T15:52:51.371Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"4f935402-4623-4a9f-8665-daa71408a900","filter labels":["test-ns-rhs44",{"ray.io/cluster":"raycluster-auth-token","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:52:51.371Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"4f935402-4623-4a9f-8665-daa71408a900","filter labels":["test-ns-rhs44",{"ray.io/cluster":"raycluster-auth-token","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:52:51.371Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"4f935402-4623-4a9f-8665-daa71408a900","filter labels":["test-ns-rhs44",{"ray.io/cluster":"raycluster-auth-token","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:52:51.371Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"4f935402-4623-4a9f-8665-daa71408a900","name":"raycluster-auth-token","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:52:51Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-auth-token-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:52:51Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:52:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:52:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:52:51Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:52:51.424Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"4f935402-4623-4a9f-8665-daa71408a900","name":"raycluster-auth-token","error":"Operation cannot be fulfilled on rayclusters.ray.io \"raycluster-auth-token\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-rhs44","name":"raycluster-auth-token"}}
{"level":"info","ts":"2026-11-03T15:52:51.424Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"4f935402-4623-4a9f-8665-daa71408a900"}
{"level":"error","ts":"2026-11-03T15:52:51.424Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"4f935402-4623-4a9f-8665-daa71408a900","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster-auth-token not found: certificates.cert-manager.io \"ray-head-cert-raycluster-auth-token\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:52:51.424Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"30027814-1e13-43f9-ad9a-ba4b13098402"}
{"level":"info","ts":"2026-11-03T15:52:51.424Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"30027814-1e13-43f9-ad9a-ba4b13098402"}
{"level":"info","ts":"2026-11-03T15:52:51.424Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"30027814-1e13-43f9-ad9a-ba4b13098402","1 head service found":"raycluster-auth-token-head-svc"}
{"level":"info","ts":"2026-11-03T15:52:51.424Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"30027814-1e13-43f9-ad9a-ba4b13098402"}
{"level":"info","ts":"2026-11-03T15:52:51.429Z","logger":"controllers.NetworkPolicy","msg":"Successfully reconciled NetworkPolicies for RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"bc082e18-39b4-4a63-b506-bfc997d0a87b","cluster":"raycluster-auth-token","namespace":"test-ns-rhs44"}
{"level":"info","ts":"2026-11-03T15:52:51.430Z","logger":"authentication-controller","msg":"Service account reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"namespace":"test-ns-rhs44","name":"raycluster-auth-token","reconcileID":"9ac2e95e-a308-4e13-8e92-b780ccb860a2","name":"raycluster-auth-token-oauth-proxy-sa","operation":"created","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-11-03T15:52:51.473Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"30027814-1e13-43f9-ad9a-ba4b13098402","error":"certificate ray-head-cert-raycluster-auth-token not found: certificates.cert-manager.io \"ray-head-cert-raycluster-auth-token\" not found"}
{"level":"error","ts":"2026-11-03T15:52:51.473Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"30027814-1e13-43f9-ad9a-ba4b13098402","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster-auth-token not found: certificates.cert-manager.io \"ray-head-cert-raycluster-auth-token\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:52:51.473Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"30027814-1e13-43f9-ad9a-ba4b13098402","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster-auth-token not found: certificates.cert-manager.io \"ray-head-cert-raycluster-auth-token\" not found"}
{"level":"info","ts":"2026-11-03T15:52:51.474Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"30027814-1e13-43f9-ad9a-ba4b13098402","filter labels":["test-ns-rhs44",{"ray.io/cluster":"raycluster-auth-token","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:52:51.474Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"30027814-1e13-43f9-ad9a-ba4b13098402","filter labels":["test-ns-rhs44",{"ray.io/cluster":"raycluster-auth-token","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:52:51.474Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"30027814-1e13-43f9-ad9a-ba4b13098402","filter labels":["test-ns-rhs44",{"ray.io/cluster":"raycluster-auth-token","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:52:51.474Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"30027814-1e13-43f9-ad9a-ba4b13098402","name":"raycluster-auth-token","old status":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:52:51Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-auth-token-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:52:51Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:52:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:52:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:52:51Z","reason":"RayClusterSuspending","message":""}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:52:51.482Z","logger":"authentication-controller","msg":"ReferenceGrant reconciled (shared)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"namespace":"test-ns-rhs44","name":"raycluster-auth-token","reconcileID":"9ac2e95e-a308-4e13-8e92-b780ccb860a2","name":"kuberay-gateway-access","namespace":"test-ns-rhs44","operation":"created"}
{"level":"info","ts":"2026-11-03T15:52:51.533Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"30027814-1e13-43f9-ad9a-ba4b13098402"}
{"level":"error","ts":"2026-11-03T15:52:51.533Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"30027814-1e13-43f9-ad9a-ba4b13098402","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster-auth-token not found: certificates.cert-manager.io \"ray-head-cert-raycluster-auth-token\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:52:51.536Z","logger":"authentication-controller","msg":"HTTPRoute reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"namespace":"test-ns-rhs44","name":"raycluster-auth-token","reconcileID":"9ac2e95e-a308-4e13-8e92-b780ccb860a2","name":"test-ns-rhs44-raycluster-auth-token","operation":"created"}
{"level":"info","ts":"2026-11-03T15:52:51.544Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"5ca769fb-4e43-4e10-ae74-1bc62ad041bd"}
{"level":"info","ts":"2026-11-03T15:52:51.544Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"5ca769fb-4e43-4e10-ae74-1bc62ad041bd"}
{"level":"info","ts":"2026-11-03T15:52:51.544Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"5ca769fb-4e43-4e10-ae74-1bc62ad041bd","1 head service found":"raycluster-auth-token-head-svc"}
{"level":"info","ts":"2026-11-03T15:52:51.544Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"5ca769fb-4e43-4e10-ae74-1bc62ad041bd"}
{"level":"info","ts":"2026-11-03T15:52:51.592Z","logger":"authentication-controller","msg":"OIDC ConfigMap reconciled","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"namespace":"test-ns-rhs44","name":"raycluster-auth-token","reconcileID":"9ac2e95e-a308-4e13-8e92-b780ccb860a2","name":"kube-rbac-proxy-config-raycluster-auth-token","operation":"created"}
{"level":"info","ts":"2026-11-03T15:52:51.593Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"5ca769fb-4e43-4e10-ae74-1bc62ad041bd","error":"certificate ray-head-cert-raycluster-auth-token not found: certificates.cert-manager.io \"ray-head-cert-raycluster-auth-token\" not found"}
{"level":"error","ts":"2026-11-03T15:52:51.593Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"5ca769fb-4e43-4e10-ae74-1bc62ad041bd","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster-auth-token not found: certificates.cert-manager.io \"ray-head-cert-raycluster-auth-token\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:52:51.593Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"5ca769fb-4e43-4e10-ae74-1bc62ad041bd","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster-auth-token not found: certificates.cert-manager.io \"ray-head-cert-raycluster-auth-token\" not found"}
{"level":"info","ts":"2026-11-03T15:52:51.593Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"5ca769fb-4e43-4e10-ae74-1bc62ad041bd","filter labels":["test-ns-rhs44",{"ray.io/cluster":"raycluster-auth-token","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:52:51.593Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"5ca769fb-4e43-4e10-ae74-1bc62ad041bd","filter labels":["test-ns-rhs44",{"ray.io/cluster":"raycluster-auth-token","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:52:51.593Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"5ca769fb-4e43-4e10-ae74-1bc62ad041bd","filter labels":["test-ns-rhs44",{"ray.io/cluster":"raycluster-auth-token","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:52:51.593Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"5ca769fb-4e43-4e10-ae74-1bc62ad041bd"}
{"level":"error","ts":"2026-11-03T15:52:51.593Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"5ca769fb-4e43-4e10-ae74-1bc62ad041bd","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster-auth-token not found: certificates.cert-manager.io \"ray-head-cert-raycluster-auth-token\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:52:51.613Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"4a336790-b7e3-4d20-9190-4748eafffdb7"}
{"level":"info","ts":"2026-11-03T15:52:51.613Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"4a336790-b7e3-4d20-9190-4748eafffdb7"}
{"level":"info","ts":"2026-11-03T15:52:51.613Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"4a336790-b7e3-4d20-9190-4748eafffdb7","1 head service found":"raycluster-auth-token-head-svc"}
{"level":"info","ts":"2026-11-03T15:52:51.613Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"4a336790-b7e3-4d20-9190-4748eafffdb7"}
{"level":"error","ts":"2026-11-03T15:52:51.646Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"namespace":"test-ns-rhs44","name":"raycluster-auth-token","reconcileID":"9ac2e95e-a308-4e13-8e92-b780ccb860a2","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"raycluster-auth-token\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:167\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:52:51.646Z","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"namespace":"test-ns-rhs44","name":"raycluster-auth-token","reconcileID":"9ac2e95e-a308-4e13-8e92-b780ccb860a2"}
{"level":"error","ts":"2026-11-03T15:52:51.646Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"namespace":"test-ns-rhs44","name":"raycluster-auth-token","reconcileID":"9ac2e95e-a308-4e13-8e92-b780ccb860a2","error":"authentication resources created but failed to update AuthenticationReady condition: Operation cannot be fulfilled on rayclusters.ray.io \"raycluster-auth-token\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:52:51.646Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"namespace":"test-ns-rhs44","name":"raycluster-auth-token","reconcileID":"7ac4c53f-091b-46ef-a9bc-c89da073b051","namespacedName":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"}}
{"level":"info","ts":"2026-11-03T15:52:51.646Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"namespace":"test-ns-rhs44","name":"raycluster-auth-token","reconcileID":"7ac4c53f-091b-46ef-a9bc-c89da073b051","mode":"IntegratedOAuth","cluster":"raycluster-auth-token"}
{"level":"info","ts":"2026-11-03T15:52:51.646Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"namespace":"test-ns-rhs44","name":"raycluster-auth-token","reconcileID":"7ac4c53f-091b-46ef-a9bc-c89da073b051","cluster":"raycluster-auth-token"}
{"level":"info","ts":"2026-11-03T15:52:51.663Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"4a336790-b7e3-4d20-9190-4748eafffdb7","error":"certificate ray-head-cert-raycluster-auth-token not found: certificates.cert-manager.io \"ray-head-cert-raycluster-auth-token\" not found"}
{"level":"error","ts":"2026-11-03T15:52:51.663Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"4a336790-b7e3-4d20-9190-4748eafffdb7","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster-auth-token not found: certificates.cert-manager.io \"ray-head-cert-raycluster-auth-token\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:52:51.663Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"4a336790-b7e3-4d20-9190-4748eafffdb7","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster-auth-token not found: certificates.cert-manager.io \"ray-head-cert-raycluster-auth-token\" not found"}
{"level":"info","ts":"2026-11-03T15:52:51.663Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"4a336790-b7e3-4d20-9190-4748eafffdb7","filter labels":["test-ns-rhs44",{"ray.io/cluster":"raycluster-auth-token","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:52:51.663Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"4a336790-b7e3-4d20-9190-4748eafffdb7","filter labels":["test-ns-rhs44",{"ray.io/cluster":"raycluster-auth-token","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:52:51.663Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"4a336790-b7e3-4d20-9190-4748eafffdb7","filter labels":["test-ns-rhs44",{"ray.io/cluster":"raycluster-auth-token","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:52:51.663Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"4a336790-b7e3-4d20-9190-4748eafffdb7"}
{"level":"error","ts":"2026-11-03T15:52:51.663Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"4a336790-b7e3-4d20-9190-4748eafffdb7","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster-auth-token not found: certificates.cert-manager.io \"ray-head-cert-raycluster-auth-token\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:52:51.704Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"7013c6e7-7aa8-487d-a3d2-c6827da184d4"}
{"level":"info","ts":"2026-11-03T15:52:51.704Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"7013c6e7-7aa8-487d-a3d2-c6827da184d4"}
{"level":"info","ts":"2026-11-03T15:52:51.704Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"7013c6e7-7aa8-487d-a3d2-c6827da184d4","1 head service found":"raycluster-auth-token-head-svc"}
{"level":"info","ts":"2026-11-03T15:52:51.704Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"7013c6e7-7aa8-487d-a3d2-c6827da184d4"}
{"level":"info","ts":"2026-11-03T15:52:51.704Z","logger":"authentication-controller","msg":"Successfully reconciled authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"namespace":"test-ns-rhs44","name":"raycluster-auth-token","reconcileID":"7ac4c53f-091b-46ef-a9bc-c89da073b051","cluster":"raycluster-auth-token"}
{"level":"info","ts":"2026-11-03T15:52:51.754Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"7013c6e7-7aa8-487d-a3d2-c6827da184d4","error":"certificate ray-head-cert-raycluster-auth-token not found: certificates.cert-manager.io \"ray-head-cert-raycluster-auth-token\" not found"}
{"level":"error","ts":"2026-11-03T15:52:51.754Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"7013c6e7-7aa8-487d-a3d2-c6827da184d4","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster-auth-token not found: certificates.cert-manager.io \"ray-head-cert-raycluster-auth-token\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:52:51.754Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"7013c6e7-7aa8-487d-a3d2-c6827da184d4","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster-auth-token not found: certificates.cert-manager.io \"ray-head-cert-raycluster-auth-token\" not found"}
{"level":"info","ts":"2026-11-03T15:52:51.754Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"7013c6e7-7aa8-487d-a3d2-c6827da184d4","filter labels":["test-ns-rhs44",{"ray.io/cluster":"raycluster-auth-token","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:52:51.754Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"7013c6e7-7aa8-487d-a3d2-c6827da184d4","filter labels":["test-ns-rhs44",{"ray.io/cluster":"raycluster-auth-token","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:52:51.754Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"7013c6e7-7aa8-487d-a3d2-c6827da184d4","filter labels":["test-ns-rhs44",{"ray.io/cluster":"raycluster-auth-token","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:52:51.754Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"7013c6e7-7aa8-487d-a3d2-c6827da184d4"}
{"level":"error","ts":"2026-11-03T15:52:51.754Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"7013c6e7-7aa8-487d-a3d2-c6827da184d4","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster-auth-token not found: certificates.cert-manager.io \"ray-head-cert-raycluster-auth-token\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:52:51.835Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"cfdf4737-c1c3-4415-8a40-dd93caa7513f"}
{"level":"info","ts":"2026-11-03T15:52:51.835Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"cfdf4737-c1c3-4415-8a40-dd93caa7513f"}
{"level":"info","ts":"2026-11-03T15:52:51.835Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"cfdf4737-c1c3-4415-8a40-dd93caa7513f","1 head service found":"raycluster-auth-token-head-svc"}
{"level":"info","ts":"2026-11-03T15:52:51.835Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"cfdf4737-c1c3-4415-8a40-dd93caa7513f"}
{"level":"info","ts":"2026-11-03T15:52:51.885Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"cfdf4737-c1c3-4415-8a40-dd93caa7513f","error":"certificate ray-head-cert-raycluster-auth-token is not ready at current generation 1"}
{"level":"error","ts":"2026-11-03T15:52:51.885Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"cfdf4737-c1c3-4415-8a40-dd93caa7513f","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster-auth-token is not ready at current generation 1","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:52:51.885Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"cfdf4737-c1c3-4415-8a40-dd93caa7513f","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster-auth-token is not ready at current generation 1"}
{"level":"info","ts":"2026-11-03T15:52:51.885Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"cfdf4737-c1c3-4415-8a40-dd93caa7513f","filter labels":["test-ns-rhs44",{"ray.io/cluster":"raycluster-auth-token","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:52:51.885Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"cfdf4737-c1c3-4415-8a40-dd93caa7513f","filter labels":["test-ns-rhs44",{"ray.io/cluster":"raycluster-auth-token","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:52:51.886Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"cfdf4737-c1c3-4415-8a40-dd93caa7513f","filter labels":["test-ns-rhs44",{"ray.io/cluster":"raycluster-auth-token","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:52:51.886Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"cfdf4737-c1c3-4415-8a40-dd93caa7513f"}
{"level":"error","ts":"2026-11-03T15:52:51.886Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"cfdf4737-c1c3-4415-8a40-dd93caa7513f","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster-auth-token is not ready at current generation 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:52:52.047Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"4deedfc6-eb7b-42c3-9804-bc4f4ec66fef"}
{"level":"info","ts":"2026-11-03T15:52:52.047Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"4deedfc6-eb7b-42c3-9804-bc4f4ec66fef"}
{"level":"info","ts":"2026-11-03T15:52:52.047Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"4deedfc6-eb7b-42c3-9804-bc4f4ec66fef","1 head service found":"raycluster-auth-token-head-svc"}
{"level":"info","ts":"2026-11-03T15:52:52.047Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"4deedfc6-eb7b-42c3-9804-bc4f4ec66fef"}
{"level":"info","ts":"2026-11-03T15:52:52.065Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"4b666730-ab1e-44db-a1ae-b6eaae0322d9","certificate":"ray-head-cert-raycluster-auth-token"}
{"level":"info","ts":"2026-11-03T15:52:52.065Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"4b666730-ab1e-44db-a1ae-b6eaae0322d9"}
{"level":"info","ts":"2026-11-03T15:52:52.065Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"73e051aa-c2b8-4238-80cd-264b75acc491","cluster":"raycluster-auth-token"}
{"level":"info","ts":"2026-11-03T15:52:52.097Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"4deedfc6-eb7b-42c3-9804-bc4f4ec66fef","error":"certificate ray-head-cert-raycluster-auth-token is not ready at current generation 1"}
{"level":"error","ts":"2026-11-03T15:52:52.097Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"4deedfc6-eb7b-42c3-9804-bc4f4ec66fef","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster-auth-token is not ready at current generation 1","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:52:52.097Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"4deedfc6-eb7b-42c3-9804-bc4f4ec66fef","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster-auth-token is not ready at current generation 1"}
{"level":"info","ts":"2026-11-03T15:52:52.097Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"4deedfc6-eb7b-42c3-9804-bc4f4ec66fef","filter labels":["test-ns-rhs44",{"ray.io/cluster":"raycluster-auth-token","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:52:52.097Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"4deedfc6-eb7b-42c3-9804-bc4f4ec66fef","filter labels":["test-ns-rhs44",{"ray.io/cluster":"raycluster-auth-token","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:52:52.097Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"4deedfc6-eb7b-42c3-9804-bc4f4ec66fef","filter labels":["test-ns-rhs44",{"ray.io/cluster":"raycluster-auth-token","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:52:52.097Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"4deedfc6-eb7b-42c3-9804-bc4f4ec66fef"}
{"level":"error","ts":"2026-11-03T15:52:52.097Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"4deedfc6-eb7b-42c3-9804-bc4f4ec66fef","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster-auth-token is not ready at current generation 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:52:52.418Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"16ee92dc-7b91-4ac2-9bc1-d53bde04a0fe"}
{"level":"info","ts":"2026-11-03T15:52:52.418Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"16ee92dc-7b91-4ac2-9bc1-d53bde04a0fe"}
{"level":"info","ts":"2026-11-03T15:52:52.418Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"16ee92dc-7b91-4ac2-9bc1-d53bde04a0fe","1 head service found":"raycluster-auth-token-head-svc"}
{"level":"info","ts":"2026-11-03T15:52:52.418Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"16ee92dc-7b91-4ac2-9bc1-d53bde04a0fe"}
{"level":"info","ts":"2026-11-03T15:52:52.467Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"16ee92dc-7b91-4ac2-9bc1-d53bde04a0fe","error":"certificate ray-head-cert-raycluster-auth-token is not ready at current generation 1"}
{"level":"error","ts":"2026-11-03T15:52:52.467Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"16ee92dc-7b91-4ac2-9bc1-d53bde04a0fe","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster-auth-token is not ready at current generation 1","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:52:52.467Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"16ee92dc-7b91-4ac2-9bc1-d53bde04a0fe","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster-auth-token is not ready at current generation 1"}
{"level":"info","ts":"2026-11-03T15:52:52.468Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"16ee92dc-7b91-4ac2-9bc1-d53bde04a0fe","filter labels":["test-ns-rhs44",{"ray.io/cluster":"raycluster-auth-token","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:52:52.468Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"16ee92dc-7b91-4ac2-9bc1-d53bde04a0fe","filter labels":["test-ns-rhs44",{"ray.io/cluster":"raycluster-auth-token","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:52:52.468Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"16ee92dc-7b91-4ac2-9bc1-d53bde04a0fe","filter labels":["test-ns-rhs44",{"ray.io/cluster":"raycluster-auth-token","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:52:52.468Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"16ee92dc-7b91-4ac2-9bc1-d53bde04a0fe"}
{"level":"error","ts":"2026-11-03T15:52:52.468Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"16ee92dc-7b91-4ac2-9bc1-d53bde04a0fe","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster-auth-token is not ready at current generation 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:52:52.522Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"73e051aa-c2b8-4238-80cd-264b75acc491","certificate":"ray-head-cert-raycluster-auth-token"}
{"level":"info","ts":"2026-11-03T15:52:52.522Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"73e051aa-c2b8-4238-80cd-264b75acc491"}
{"level":"info","ts":"2026-11-03T15:52:53.108Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"5cd29527-8b9e-4889-8109-ca32e8d665ff"}
{"level":"info","ts":"2026-11-03T15:52:53.109Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"5cd29527-8b9e-4889-8109-ca32e8d665ff"}
{"level":"info","ts":"2026-11-03T15:52:53.109Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"5cd29527-8b9e-4889-8109-ca32e8d665ff","1 head service found":"raycluster-auth-token-head-svc"}
{"level":"info","ts":"2026-11-03T15:52:53.109Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"5cd29527-8b9e-4889-8109-ca32e8d665ff"}
{"level":"info","ts":"2026-11-03T15:52:53.159Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"5cd29527-8b9e-4889-8109-ca32e8d665ff","error":"certificate ray-head-cert-raycluster-auth-token is not ready at current generation 1"}
{"level":"error","ts":"2026-11-03T15:52:53.159Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"5cd29527-8b9e-4889-8109-ca32e8d665ff","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster-auth-token is not ready at current generation 1","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:52:53.159Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"5cd29527-8b9e-4889-8109-ca32e8d665ff","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster-auth-token is not ready at current generation 1"}
{"level":"info","ts":"2026-11-03T15:52:53.159Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"5cd29527-8b9e-4889-8109-ca32e8d665ff","filter labels":["test-ns-rhs44",{"ray.io/cluster":"raycluster-auth-token","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:52:53.159Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"5cd29527-8b9e-4889-8109-ca32e8d665ff","filter labels":["test-ns-rhs44",{"ray.io/cluster":"raycluster-auth-token","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:52:53.159Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"5cd29527-8b9e-4889-8109-ca32e8d665ff","filter labels":["test-ns-rhs44",{"ray.io/cluster":"raycluster-auth-token","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:52:53.159Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"5cd29527-8b9e-4889-8109-ca32e8d665ff"}
{"level":"error","ts":"2026-11-03T15:52:53.159Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"5cd29527-8b9e-4889-8109-ca32e8d665ff","error":"mTLS secrets not ready: certificate ray-head-cert-raycluster-auth-token is not ready at current generation 1","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:52:53.727Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"aebb731e-e83b-4bb0-adcd-0fe634e65f12"}
{"level":"info","ts":"2026-11-03T15:52:53.727Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"aebb731e-e83b-4bb0-adcd-0fe634e65f12"}
{"level":"info","ts":"2026-11-03T15:52:53.727Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"aebb731e-e83b-4bb0-adcd-0fe634e65f12","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:52:53.727Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"aebb731e-e83b-4bb0-adcd-0fe634e65f12","Found 1 head Pod":"ray-cluster-head-jhq58","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:52:53.727Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"aebb731e-e83b-4bb0-adcd-0fe634e65f12","head Pod":"ray-cluster-head-jhq58","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-jhq58. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:52:53.727Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"aebb731e-e83b-4bb0-adcd-0fe634e65f12","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-11-03T15:52:53.727Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"aebb731e-e83b-4bb0-adcd-0fe634e65f12","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:52:53.727Z","logger":"controllers.RayCluster","msg":"Deleting pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"aebb731e-e83b-4bb0-adcd-0fe634e65f12","namespace":"test-ns-zjpgs","name":"ray-cluster-small-group-worker-lc796"}
{"level":"info","ts":"2026-11-03T15:52:53.776Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"aebb731e-e83b-4bb0-adcd-0fe634e65f12","The worker Pod has already been deleted":"ray-cluster-small-group-worker-lc796"}
{"level":"info","ts":"2026-11-03T15:52:53.776Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"aebb731e-e83b-4bb0-adcd-0fe634e65f12","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-11-03T15:52:53.776Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"aebb731e-e83b-4bb0-adcd-0fe634e65f12","group":"small-group","indices":{}}
{"level":"info","ts":"2026-11-03T15:52:53.776Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"aebb731e-e83b-4bb0-adcd-0fe634e65f12","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:52:53.777Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"aebb731e-e83b-4bb0-adcd-0fe634e65f12","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:52:53.777Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"aebb731e-e83b-4bb0-adcd-0fe634e65f12","seconds":300}
{"level":"info","ts":"2026-11-03T15:52:53.904Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"ef3b9684-12ff-43af-bf99-542acc0f76e3"}
{"level":"info","ts":"2026-11-03T15:52:53.904Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"ef3b9684-12ff-43af-bf99-542acc0f76e3"}
{"level":"info","ts":"2026-11-03T15:52:53.904Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"ef3b9684-12ff-43af-bf99-542acc0f76e3","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:52:53.904Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"ef3b9684-12ff-43af-bf99-542acc0f76e3","Found 1 head Pod":"ray-cluster-head-jhq58","Pod status":"Running","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:52:53.904Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"ef3b9684-12ff-43af-bf99-542acc0f76e3","head Pod":"ray-cluster-head-jhq58","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod ray-cluster-head-jhq58. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:52:53.904Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"ef3b9684-12ff-43af-bf99-542acc0f76e3","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-11-03T15:52:53.904Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"ef3b9684-12ff-43af-bf99-542acc0f76e3","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:52:53.904Z","logger":"controllers.RayCluster","msg":"Deleting pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"ef3b9684-12ff-43af-bf99-542acc0f76e3","namespace":"test-ns-zjpgs","name":"ray-cluster-small-group-worker-lc796"}
{"level":"info","ts":"2026-11-03T15:52:53.953Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"ef3b9684-12ff-43af-bf99-542acc0f76e3","The worker Pod has already been deleted":"ray-cluster-small-group-worker-lc796"}
{"level":"info","ts":"2026-11-03T15:52:53.953Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"ef3b9684-12ff-43af-bf99-542acc0f76e3","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-11-03T15:52:53.953Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"ef3b9684-12ff-43af-bf99-542acc0f76e3","group":"small-group","indices":{}}
{"level":"info","ts":"2026-11-03T15:52:53.953Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"ef3b9684-12ff-43af-bf99-542acc0f76e3","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:52:53.954Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"ef3b9684-12ff-43af-bf99-542acc0f76e3","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:52:53.954Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"ef3b9684-12ff-43af-bf99-542acc0f76e3","seconds":300}
{"level":"info","ts":"2026-11-03T15:52:54.440Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"e0bd0663-8359-4fc3-9953-6117c5c4b226"}
{"level":"info","ts":"2026-11-03T15:52:54.440Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"e0bd0663-8359-4fc3-9953-6117c5c4b226"}
{"level":"info","ts":"2026-11-03T15:52:54.440Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"e0bd0663-8359-4fc3-9953-6117c5c4b226","1 head service found":"raycluster-auth-token-head-svc"}
{"level":"info","ts":"2026-11-03T15:52:54.440Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"e0bd0663-8359-4fc3-9953-6117c5c4b226"}
{"level":"info","ts":"2026-11-03T15:52:54.542Z","logger":"controllers.RayCluster","msg":"Detected authentication mode for pod creation","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"e0bd0663-8359-4fc3-9953-6117c5c4b226","mode":"IntegratedOAuth","cluster":"raycluster-auth-token"}
{"level":"info","ts":"2026-11-03T15:52:54.542Z","logger":"controllers.RayCluster","msg":"Injecting OIDC proxy sidecar (kube-rbac-proxy)","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"e0bd0663-8359-4fc3-9953-6117c5c4b226","cluster":"raycluster-auth-token","mode":"IntegratedOAuth"}
{"level":"info","ts":"2026-11-03T15:52:54.542Z","logger":"controllers.RayCluster","msg":"Authentication sidecar injected successfully","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"e0bd0663-8359-4fc3-9953-6117c5c4b226","cluster":"raycluster-auth-token","authType":"IntegratedOAuth","serviceAccount":"raycluster-auth-token-oauth-proxy-sa","containerCount":2}
{"level":"info","ts":"2026-11-03T15:52:54.542Z","logger":"controllers.RayCluster","msg":"head pod labels","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"e0bd0663-8359-4fc3-9953-6117c5c4b226","labels":{"app.kubernetes.io/created-by":"kuberay-operator","app.kubernetes.io/name":"kuberay","ray.io/cluster":"raycluster-auth-token","ray.io/group":"headgroup","ray.io/identifier":"raycluster-auth-token-head","ray.io/is-ray-node":"yes","ray.io/node-type":"head"}}
{"level":"info","ts":"2026-11-03T15:52:54.542Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"e0bd0663-8359-4fc3-9953-6117c5c4b226","nodeType":"head","rayStartParams":{"block":"true","dashboard-agent-listen-port":"52365","dashboard-host":"0.0.0.0","metrics-export-port":"8080"},"Ray container resource":{"limits":{"cpu":"2","memory":"10G"},"requests":{"cpu":"500m","memory":"6G"}}}
{"level":"info","ts":"2026-11-03T15:52:54.542Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"e0bd0663-8359-4fc3-9953-6117c5c4b226","rayStartCmd":"ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-11-03T15:52:54.542Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"e0bd0663-8359-4fc3-9953-6117c5c4b226","rayNodeType":"head","generatedCmd":"ulimit -n ${RAY_START_ULIMIT_OPEN_FILES:-65536}; ray start --head  --block  --dashboard-agent-listen-port=52365  --dashboard-host=0.0.0.0  --memory=10000000000  --metrics-export-port=8080  --num-cpus=2 "}
{"level":"info","ts":"2026-11-03T15:52:54.542Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"e0bd0663-8359-4fc3-9953-6117c5c4b226","enabled":true}
{"level":"info","ts":"2026-11-03T15:52:54.638Z","logger":"controllers.RayCluster","msg":"Created head Pod for RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"e0bd0663-8359-4fc3-9953-6117c5c4b226","name":"raycluster-auth-token-head-985vp"}
{"level":"info","ts":"2026-11-03T15:52:54.638Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"e0bd0663-8359-4fc3-9953-6117c5c4b226","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:52:54.638Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"e0bd0663-8359-4fc3-9953-6117c5c4b226","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:52:54.638Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"e0bd0663-8359-4fc3-9953-6117c5c4b226","workerReplicas":1,"NumOfHosts":1,"runningPods":0,"diff":1}
{"level":"info","ts":"2026-11-03T15:52:54.638Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"e0bd0663-8359-4fc3-9953-6117c5c4b226","group":"small-group","indices":{}}
{"level":"info","ts":"2026-11-03T15:52:54.638Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"e0bd0663-8359-4fc3-9953-6117c5c4b226","Number workers to add":1,"Worker group":"small-group"}
{"level":"info","ts":"2026-11-03T15:52:54.738Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"e0bd0663-8359-4fc3-9953-6117c5c4b226","creating worker for group":"small-group","index":0,"total":1,"replicaIndex":0}
{"level":"info","ts":"2026-11-03T15:52:54.738Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"e0bd0663-8359-4fc3-9953-6117c5c4b226","nodeType":"worker","rayStartParams":{"address":"raycluster-auth-token-head-svc.test-ns-rhs44.svc.cluster.local:6379","block":"true","dashboard-agent-listen-port":"52365","metrics-export-port":"8080","num-cpus":"1"},"Ray container resource":{"limits":{"cpu":"1","memory":"3G"},"requests":{"cpu":"500m","memory":"1G"}}}
{"level":"info","ts":"2026-11-03T15:52:54.738Z","logger":"controllers.RayCluster","msg":"generateRayStartCommand","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"e0bd0663-8359-4fc3-9953-6117c5c4b226","rayStartCmd":"ray start  --address=raycluster-auth-token-head-svc.test-ns-rhs44.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-11-03T15:52:54.738Z","logger":"controllers.RayCluster","msg":"BuildPod","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"e0bd0663-8359-4fc3-9953-6117c5c4b226","rayNodeType":"worker","generatedCmd":"ulimit -n ${RAY_START_ULIMIT_OPEN_FILES:-65536}; ray start  --address=raycluster-auth-token-head-svc.test-ns-rhs44.svc.cluster.local:6379  --block  --dashboard-agent-listen-port=52365  --memory=3000000000  --metrics-export-port=8080  --num-cpus=1 "}
{"level":"info","ts":"2026-11-03T15:52:54.738Z","logger":"controllers.RayCluster","msg":"Probes injection feature flag","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"e0bd0663-8359-4fc3-9953-6117c5c4b226","enabled":true}
{"level":"info","ts":"2026-11-03T15:52:54.800Z","logger":"controllers.RayCluster","msg":"Created worker Pod for RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"e0bd0663-8359-4fc3-9953-6117c5c4b226","name":"raycluster-auth-token-small-group-worker-jsd4f"}
{"level":"info","ts":"2026-11-03T15:52:54.800Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"e0bd0663-8359-4fc3-9953-6117c5c4b226","podName":"raycluster-auth-token-head-985vp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:52:54.800Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"e0bd0663-8359-4fc3-9953-6117c5c4b226","name":"raycluster-auth-token","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:52:51Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-auth-token-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:52:51Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:52:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:52:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:52:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:52:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:52:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"raycluster-auth-token-head-985vp","serviceName":"raycluster-auth-token-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:52:51Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:52:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:52:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:52:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:52:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:52:54.860Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"4001f316-f8be-4b29-96cf-de4354d421a2"}
{"level":"info","ts":"2026-11-03T15:52:54.860Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"4001f316-f8be-4b29-96cf-de4354d421a2"}
{"level":"info","ts":"2026-11-03T15:52:54.860Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"4001f316-f8be-4b29-96cf-de4354d421a2","1 head service found":"raycluster-auth-token-head-svc"}
{"level":"info","ts":"2026-11-03T15:52:54.860Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"328e99af-7f1c-4ee6-ac3d-890307e3017f","cluster":"raycluster-auth-token"}
{"level":"info","ts":"2026-11-03T15:52:54.860Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"4001f316-f8be-4b29-96cf-de4354d421a2","Found 1 head Pod":"raycluster-auth-token-head-985vp","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:52:54.860Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"4001f316-f8be-4b29-96cf-de4354d421a2","head Pod":"raycluster-auth-token-head-985vp","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-token-head-985vp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:52:54.860Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"4001f316-f8be-4b29-96cf-de4354d421a2","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:52:54.860Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"4001f316-f8be-4b29-96cf-de4354d421a2","worker Pod":"raycluster-auth-token-small-group-worker-jsd4f","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-token-small-group-worker-jsd4f. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:52:54.860Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"4001f316-f8be-4b29-96cf-de4354d421a2","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:52:54.860Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"4001f316-f8be-4b29-96cf-de4354d421a2","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:52:54.860Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"4001f316-f8be-4b29-96cf-de4354d421a2","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:52:54.860Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"4001f316-f8be-4b29-96cf-de4354d421a2","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:52:54.860Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"4001f316-f8be-4b29-96cf-de4354d421a2","podName":"raycluster-auth-token-head-985vp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:52:54.860Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"4001f316-f8be-4b29-96cf-de4354d421a2","podName":"raycluster-auth-token-head-985vp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:52:54.860Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"4001f316-f8be-4b29-96cf-de4354d421a2","name":"raycluster-auth-token","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:52:51Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"raycluster-auth-token-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:52:51Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:52:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:52:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:52:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:52:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:52:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"raycluster-auth-token-head-985vp","serviceName":"raycluster-auth-token-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:52:51Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:52:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:52:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:52:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:52:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:52:54.914Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"4001f316-f8be-4b29-96cf-de4354d421a2","name":"raycluster-auth-token","error":"Operation cannot be fulfilled on rayclusters.ray.io \"raycluster-auth-token\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-rhs44","name":"raycluster-auth-token"}}
{"level":"info","ts":"2026-11-03T15:52:54.914Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"4001f316-f8be-4b29-96cf-de4354d421a2"}
{"level":"error","ts":"2026-11-03T15:52:54.914Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"4001f316-f8be-4b29-96cf-de4354d421a2","error":"Operation cannot be fulfilled on rayclusters.ray.io \"raycluster-auth-token\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:52:54.914Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"1754c630-53bc-46b9-b87d-fdae884de7fb"}
{"level":"info","ts":"2026-11-03T15:52:54.914Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"1754c630-53bc-46b9-b87d-fdae884de7fb"}
{"level":"info","ts":"2026-11-03T15:52:54.914Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"1754c630-53bc-46b9-b87d-fdae884de7fb","1 head service found":"raycluster-auth-token-head-svc"}
{"level":"info","ts":"2026-11-03T15:52:54.915Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"1754c630-53bc-46b9-b87d-fdae884de7fb","Found 1 head Pod":"raycluster-auth-token-head-985vp","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:52:54.915Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"1754c630-53bc-46b9-b87d-fdae884de7fb","head Pod":"raycluster-auth-token-head-985vp","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-token-head-985vp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:52:54.915Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"1754c630-53bc-46b9-b87d-fdae884de7fb","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:52:54.915Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"1754c630-53bc-46b9-b87d-fdae884de7fb","worker Pod":"raycluster-auth-token-small-group-worker-jsd4f","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-token-small-group-worker-jsd4f. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:52:54.915Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"1754c630-53bc-46b9-b87d-fdae884de7fb","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:52:54.915Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"1754c630-53bc-46b9-b87d-fdae884de7fb","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:52:54.915Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"1754c630-53bc-46b9-b87d-fdae884de7fb","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:52:54.915Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"1754c630-53bc-46b9-b87d-fdae884de7fb","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:52:54.915Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"1754c630-53bc-46b9-b87d-fdae884de7fb","podName":"raycluster-auth-token-head-985vp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:52:54.915Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"1754c630-53bc-46b9-b87d-fdae884de7fb","podName":"raycluster-auth-token-head-985vp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:52:54.915Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"1754c630-53bc-46b9-b87d-fdae884de7fb","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:52:54.915Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"1754c630-53bc-46b9-b87d-fdae884de7fb","seconds":300}
{"level":"info","ts":"2026-11-03T15:52:54.931Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"7bd37a91-acf6-4605-9df0-020c471005c9"}
{"level":"info","ts":"2026-11-03T15:52:54.931Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"7bd37a91-acf6-4605-9df0-020c471005c9"}
{"level":"info","ts":"2026-11-03T15:52:54.931Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"7bd37a91-acf6-4605-9df0-020c471005c9","1 head service found":"raycluster-auth-token-head-svc"}
{"level":"info","ts":"2026-11-03T15:52:54.931Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"7bd37a91-acf6-4605-9df0-020c471005c9","Found 1 head Pod":"raycluster-auth-token-head-985vp","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:52:54.932Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"7bd37a91-acf6-4605-9df0-020c471005c9","head Pod":"raycluster-auth-token-head-985vp","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-token-head-985vp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:52:54.932Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"7bd37a91-acf6-4605-9df0-020c471005c9","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:52:54.932Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"7bd37a91-acf6-4605-9df0-020c471005c9","worker Pod":"raycluster-auth-token-small-group-worker-jsd4f","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-token-small-group-worker-jsd4f. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:52:54.932Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"7bd37a91-acf6-4605-9df0-020c471005c9","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:52:54.932Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"7bd37a91-acf6-4605-9df0-020c471005c9","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:52:54.932Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"7bd37a91-acf6-4605-9df0-020c471005c9","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:52:54.932Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"7bd37a91-acf6-4605-9df0-020c471005c9","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:52:54.932Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"7bd37a91-acf6-4605-9df0-020c471005c9","podName":"raycluster-auth-token-small-group-worker-jsd4f","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:52:54.932Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"7bd37a91-acf6-4605-9df0-020c471005c9","podName":"raycluster-auth-token-small-group-worker-jsd4f","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:52:54.932Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"7bd37a91-acf6-4605-9df0-020c471005c9","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:52:54.932Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"7bd37a91-acf6-4605-9df0-020c471005c9","seconds":300}
{"level":"info","ts":"2026-11-03T15:52:55.369Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"693b4f8b-a2dd-433d-ba19-4fa964d7c57f"}
{"level":"info","ts":"2026-11-03T15:52:55.369Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"693b4f8b-a2dd-433d-ba19-4fa964d7c57f"}
{"level":"info","ts":"2026-11-03T15:52:55.369Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"693b4f8b-a2dd-433d-ba19-4fa964d7c57f","1 head service found":"raycluster-auth-token-head-svc"}
{"level":"info","ts":"2026-11-03T15:52:55.369Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"693b4f8b-a2dd-433d-ba19-4fa964d7c57f","Found 1 head Pod":"raycluster-auth-token-head-985vp","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:52:55.369Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"693b4f8b-a2dd-433d-ba19-4fa964d7c57f","head Pod":"raycluster-auth-token-head-985vp","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-token-head-985vp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:52:55.369Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"693b4f8b-a2dd-433d-ba19-4fa964d7c57f","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:52:55.369Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"693b4f8b-a2dd-433d-ba19-4fa964d7c57f","worker Pod":"raycluster-auth-token-small-group-worker-jsd4f","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-token-small-group-worker-jsd4f. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:52:55.369Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"693b4f8b-a2dd-433d-ba19-4fa964d7c57f","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:52:55.369Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"693b4f8b-a2dd-433d-ba19-4fa964d7c57f","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:52:55.369Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"693b4f8b-a2dd-433d-ba19-4fa964d7c57f","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:52:55.369Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"693b4f8b-a2dd-433d-ba19-4fa964d7c57f","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:52:55.369Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"693b4f8b-a2dd-433d-ba19-4fa964d7c57f","podName":"raycluster-auth-token-head-985vp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:52:55.369Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"693b4f8b-a2dd-433d-ba19-4fa964d7c57f","podName":"raycluster-auth-token-head-985vp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:52:55.369Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"693b4f8b-a2dd-433d-ba19-4fa964d7c57f","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:52:55.369Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"693b4f8b-a2dd-433d-ba19-4fa964d7c57f","seconds":300}
{"level":"info","ts":"2026-11-03T15:52:55.572Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"1e421c80-40e7-4410-901d-48deaea10d61"}
{"level":"info","ts":"2026-11-03T15:52:55.572Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"1e421c80-40e7-4410-901d-48deaea10d61"}
{"level":"info","ts":"2026-11-03T15:52:55.572Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"1e421c80-40e7-4410-901d-48deaea10d61","1 head service found":"raycluster-auth-token-head-svc"}
{"level":"info","ts":"2026-11-03T15:52:55.572Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"1e421c80-40e7-4410-901d-48deaea10d61","Found 1 head Pod":"raycluster-auth-token-head-985vp","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:52:55.572Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"1e421c80-40e7-4410-901d-48deaea10d61","head Pod":"raycluster-auth-token-head-985vp","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-token-head-985vp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:52:55.572Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"1e421c80-40e7-4410-901d-48deaea10d61","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:52:55.572Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"1e421c80-40e7-4410-901d-48deaea10d61","worker Pod":"raycluster-auth-token-small-group-worker-jsd4f","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-token-small-group-worker-jsd4f. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:52:55.572Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"1e421c80-40e7-4410-901d-48deaea10d61","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:52:55.572Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"1e421c80-40e7-4410-901d-48deaea10d61","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:52:55.572Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"1e421c80-40e7-4410-901d-48deaea10d61","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:52:55.572Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"1e421c80-40e7-4410-901d-48deaea10d61","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:52:55.572Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"1e421c80-40e7-4410-901d-48deaea10d61","podName":"raycluster-auth-token-head-985vp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:52:55.572Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"1e421c80-40e7-4410-901d-48deaea10d61","podName":"raycluster-auth-token-head-985vp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:52:55.572Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"1e421c80-40e7-4410-901d-48deaea10d61","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:52:55.572Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"1e421c80-40e7-4410-901d-48deaea10d61","seconds":300}
{"level":"info","ts":"2026-11-03T15:52:55.580Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"616abc15-9cc9-4d4f-805e-afdb832c3ca7"}
{"level":"info","ts":"2026-11-03T15:52:55.580Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"616abc15-9cc9-4d4f-805e-afdb832c3ca7"}
{"level":"info","ts":"2026-11-03T15:52:55.580Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"616abc15-9cc9-4d4f-805e-afdb832c3ca7","1 head service found":"raycluster-auth-token-head-svc"}
{"level":"info","ts":"2026-11-03T15:52:55.580Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"616abc15-9cc9-4d4f-805e-afdb832c3ca7","Found 1 head Pod":"raycluster-auth-token-head-985vp","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:52:55.580Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"616abc15-9cc9-4d4f-805e-afdb832c3ca7","head Pod":"raycluster-auth-token-head-985vp","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-token-head-985vp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:52:55.580Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"616abc15-9cc9-4d4f-805e-afdb832c3ca7","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:52:55.580Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"616abc15-9cc9-4d4f-805e-afdb832c3ca7","worker Pod":"raycluster-auth-token-small-group-worker-jsd4f","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-token-small-group-worker-jsd4f. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:52:55.580Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"616abc15-9cc9-4d4f-805e-afdb832c3ca7","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:52:55.580Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"616abc15-9cc9-4d4f-805e-afdb832c3ca7","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:52:55.580Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"616abc15-9cc9-4d4f-805e-afdb832c3ca7","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:52:55.580Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"616abc15-9cc9-4d4f-805e-afdb832c3ca7","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:52:55.580Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"616abc15-9cc9-4d4f-805e-afdb832c3ca7","podName":"raycluster-auth-token-head-985vp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:52:55.580Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"616abc15-9cc9-4d4f-805e-afdb832c3ca7","podName":"raycluster-auth-token-head-985vp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:52:55.580Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"616abc15-9cc9-4d4f-805e-afdb832c3ca7","name":"raycluster-auth-token","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:52:54Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podName":"raycluster-auth-token-head-985vp","serviceName":"raycluster-auth-token-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:52:51Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:52:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:52:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:52:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:52:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:52:55Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.59","serviceIP":"10.134.0.59","podName":"raycluster-auth-token-head-985vp","serviceName":"raycluster-auth-token-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:52:51Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:52:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:52:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:52:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:52:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:52:55.685Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"293ef857-70d5-4f9c-9873-192546913c45","cluster":"raycluster-auth-token"}
{"level":"info","ts":"2026-11-03T15:52:55.994Z","logger":"controllers.RayClusterMTLS","msg":"Updating head certificate","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"293ef857-70d5-4f9c-9873-192546913c45","certificate":"ray-head-cert-raycluster-auth-token","ipAddresses":["10.134.0.59","127.0.0.1"]}
{"level":"info","ts":"2026-11-03T15:52:56.243Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"281daa14-b631-41aa-96b8-aa5b1af0a3eb"}
{"level":"info","ts":"2026-11-03T15:52:56.243Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"281daa14-b631-41aa-96b8-aa5b1af0a3eb"}
{"level":"info","ts":"2026-11-03T15:52:56.243Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"281daa14-b631-41aa-96b8-aa5b1af0a3eb","1 head service found":"raycluster-auth-token-head-svc"}
{"level":"info","ts":"2026-11-03T15:52:56.243Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"281daa14-b631-41aa-96b8-aa5b1af0a3eb","Found 1 head Pod":"raycluster-auth-token-head-985vp","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:52:56.243Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"281daa14-b631-41aa-96b8-aa5b1af0a3eb","head Pod":"raycluster-auth-token-head-985vp","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-token-head-985vp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:52:56.243Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"281daa14-b631-41aa-96b8-aa5b1af0a3eb","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:52:56.243Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"281daa14-b631-41aa-96b8-aa5b1af0a3eb","worker Pod":"raycluster-auth-token-small-group-worker-jsd4f","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-token-small-group-worker-jsd4f. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:52:56.243Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"281daa14-b631-41aa-96b8-aa5b1af0a3eb","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:52:56.243Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"281daa14-b631-41aa-96b8-aa5b1af0a3eb","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:52:56.243Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"281daa14-b631-41aa-96b8-aa5b1af0a3eb","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:52:56.243Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"281daa14-b631-41aa-96b8-aa5b1af0a3eb","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:52:56.243Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"281daa14-b631-41aa-96b8-aa5b1af0a3eb","podName":"raycluster-auth-token-small-group-worker-jsd4f","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:52:56.244Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"281daa14-b631-41aa-96b8-aa5b1af0a3eb","podName":"raycluster-auth-token-small-group-worker-jsd4f","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:52:56.244Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"281daa14-b631-41aa-96b8-aa5b1af0a3eb","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:52:56.244Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"281daa14-b631-41aa-96b8-aa5b1af0a3eb","seconds":300}
{"level":"info","ts":"2026-11-03T15:52:56.408Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"293ef857-70d5-4f9c-9873-192546913c45","certificate":"ray-head-cert-raycluster-auth-token"}
{"level":"info","ts":"2026-11-03T15:52:56.409Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"293ef857-70d5-4f9c-9873-192546913c45"}
{"level":"info","ts":"2026-11-03T15:53:18.439Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"082f7fcf-9ecd-4005-9829-50cf56cd459a","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:53:18.741Z","logger":"controllers.RayClusterMTLS","msg":"Updating head certificate","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"082f7fcf-9ecd-4005-9829-50cf56cd459a","certificate":"ray-head-cert-ray-cluster","ipAddresses":["127.0.0.1"]}
{"level":"error","ts":"2026-11-03T15:53:18.789Z","logger":"events","msg":"Server rejected event (will not retry!)","event":{"namespace":"test-ns-zjpgs","name":"ray-cluster.18e493ed3d0e1593"},"error":"events.events.k8s.io \"ray-cluster.18e493ed3d0e1593\" is forbidden: unable to create new content in namespace test-ns-zjpgs because it is being terminated","stacktrace":"k8s.io/client-go/tools/events.recordEvent\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:270\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).attemptRecording\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:211\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).recordToSink.func1\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:200"}
{"level":"info","ts":"2026-11-03T15:53:19.162Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"082f7fcf-9ecd-4005-9829-50cf56cd459a","certificate":"ray-head-cert-ray-cluster"}
{"level":"info","ts":"2026-11-03T15:53:19.162Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"082f7fcf-9ecd-4005-9829-50cf56cd459a"}
{"level":"info","ts":"2026-11-03T15:53:24.645Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"1ff99923-9bea-488c-9850-f1d96eae0189"}
{"level":"info","ts":"2026-11-03T15:53:24.645Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"1ff99923-9bea-488c-9850-f1d96eae0189"}
{"level":"info","ts":"2026-11-03T15:53:24.645Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"1ff99923-9bea-488c-9850-f1d96eae0189","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:53:24.645Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"1ff99923-9bea-488c-9850-f1d96eae0189","Found 1 head Pod":"ray-cluster-head-jhq58","Pod status":"Failed","Pod status reason":"","Pod restart policy":"Never","Ray container terminated status":"&ContainerStateTerminated{ExitCode:1,Signal:0,Reason:Error,Message:,StartedAt:2026-11-03 15:48:27 +0000 UTC,FinishedAt:2026-11-03 15:52:57 +0000 UTC,ContainerID:cri-o://5e338eec99f7a4a77b039f9b0ef50a1d1010cf9c8c4c2ff1fd45d1ecc6dc821c,}"}
{"level":"info","ts":"2026-11-03T15:53:24.645Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"1ff99923-9bea-488c-9850-f1d96eae0189","head Pod":"ray-cluster-head-jhq58","shouldDelete":true,"reason":"The head Pod ray-cluster-head-jhq58 status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary."}
{"level":"error","ts":"2026-11-03T15:53:24.699Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"1ff99923-9bea-488c-9850-f1d96eae0189","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"The head Pod ray-cluster-head-jhq58 status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary.","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:53:24.699Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"1ff99923-9bea-488c-9850-f1d96eae0189","error":"The head Pod ray-cluster-head-jhq58 status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary."}
{"level":"info","ts":"2026-11-03T15:53:24.699Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"1ff99923-9bea-488c-9850-f1d96eae0189","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:52:47Z","stateTransitionTimes":{"ready":"2026-11-03T15:48:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"ray-cluster-head-jhq58","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-11-03T15:46:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:48:44Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:48:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":5},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:53:24Z","stateTransitionTimes":{"ready":"2026-11-03T15:48:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"ray-cluster-head-jhq58","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-11-03T15:46:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:53:24Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:48:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":5}}
{"level":"error","ts":"2026-11-03T15:53:24.748Z","logger":"events","msg":"Server rejected event (will not retry!)","event":{"namespace":"test-ns-zjpgs","name":"ray-cluster.18e493eea02ead79"},"error":"events.events.k8s.io \"ray-cluster.18e493eea02ead79\" is forbidden: unable to create new content in namespace test-ns-zjpgs because it is being terminated","stacktrace":"k8s.io/client-go/tools/events.recordEvent\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:270\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).attemptRecording\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:211\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).recordToSink.func1\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:200"}
{"level":"info","ts":"2026-11-03T15:53:24.760Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"1ff99923-9bea-488c-9850-f1d96eae0189"}
{"level":"error","ts":"2026-11-03T15:53:24.760Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"1ff99923-9bea-488c-9850-f1d96eae0189","error":"The head Pod ray-cluster-head-jhq58 status is Failed which is a terminal state. KubeRay will delete the Pod and create new Pods in the next reconciliation if necessary.","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:53:24.761Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"40bdddb7-38fe-4841-aec2-fc45112527a0","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:53:24.765Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"60c587a5-cfa0-4cd4-a457-d563f82d380a"}
{"level":"info","ts":"2026-11-03T15:53:24.765Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"60c587a5-cfa0-4cd4-a457-d563f82d380a"}
{"level":"info","ts":"2026-11-03T15:53:24.765Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"60c587a5-cfa0-4cd4-a457-d563f82d380a","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:53:24.766Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"60c587a5-cfa0-4cd4-a457-d563f82d380a","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-11-03T15:53:24.766Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"60c587a5-cfa0-4cd4-a457-d563f82d380a","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-11-03T15:53:24.766Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"60c587a5-cfa0-4cd4-a457-d563f82d380a","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:53:24.766Z","logger":"controllers.RayCluster","msg":"Deleting pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"60c587a5-cfa0-4cd4-a457-d563f82d380a","namespace":"test-ns-zjpgs","name":"ray-cluster-small-group-worker-lc796"}
{"level":"info","ts":"2026-11-03T15:53:24.815Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"60c587a5-cfa0-4cd4-a457-d563f82d380a","The worker Pod has already been deleted":"ray-cluster-small-group-worker-lc796"}
{"level":"info","ts":"2026-11-03T15:53:24.815Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"60c587a5-cfa0-4cd4-a457-d563f82d380a","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-11-03T15:53:24.815Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"60c587a5-cfa0-4cd4-a457-d563f82d380a","group":"small-group","indices":{}}
{"level":"info","ts":"2026-11-03T15:53:24.815Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"60c587a5-cfa0-4cd4-a457-d563f82d380a","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:53:24.815Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"60c587a5-cfa0-4cd4-a457-d563f82d380a","podName":"ray-cluster-head-jhq58","pod Status.Phase":"Failed"}
{"level":"info","ts":"2026-11-03T15:53:24.815Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"60c587a5-cfa0-4cd4-a457-d563f82d380a","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:53:24.815Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"60c587a5-cfa0-4cd4-a457-d563f82d380a","seconds":300}
{"level":"info","ts":"2026-11-03T15:53:25.368Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"40bdddb7-38fe-4841-aec2-fc45112527a0","certificate":"ray-head-cert-ray-cluster"}
{"level":"info","ts":"2026-11-03T15:53:25.368Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"40bdddb7-38fe-4841-aec2-fc45112527a0"}
{"level":"error","ts":"2026-11-03T15:53:25.417Z","logger":"events","msg":"Server rejected event (will not retry!)","event":{"namespace":"test-ns-zjpgs","name":"ray-cluster.18e493eec80f9d34"},"error":"events.events.k8s.io \"ray-cluster.18e493eec80f9d34\" is forbidden: unable to create new content in namespace test-ns-zjpgs because it is being terminated","stacktrace":"k8s.io/client-go/tools/events.recordEvent\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:270\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).attemptRecording\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:211\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).recordToSink.func1\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:200"}
{"level":"info","ts":"2026-11-03T15:53:25.461Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"a593d80b-56bc-421e-aeeb-3d10cb3696ad"}
{"level":"info","ts":"2026-11-03T15:53:25.461Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"a593d80b-56bc-421e-aeeb-3d10cb3696ad"}
{"level":"info","ts":"2026-11-03T15:53:25.461Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"a593d80b-56bc-421e-aeeb-3d10cb3696ad","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:53:25.462Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"a593d80b-56bc-421e-aeeb-3d10cb3696ad","Expectation":"NotSatisfiedHeadExpectations, reconcile head later"}
{"level":"info","ts":"2026-11-03T15:53:25.462Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"a593d80b-56bc-421e-aeeb-3d10cb3696ad","desired workerReplicas (always adhering to minReplicas/maxReplica)":0,"worker group":"small-group","maxReplicas":3,"minReplicas":0,"replicas":0}
{"level":"info","ts":"2026-11-03T15:53:25.462Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"a593d80b-56bc-421e-aeeb-3d10cb3696ad","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:53:25.462Z","logger":"controllers.RayCluster","msg":"Deleting pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"a593d80b-56bc-421e-aeeb-3d10cb3696ad","namespace":"test-ns-zjpgs","name":"ray-cluster-small-group-worker-lc796"}
{"level":"info","ts":"2026-11-03T15:53:25.514Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"a593d80b-56bc-421e-aeeb-3d10cb3696ad","The worker Pod has already been deleted":"ray-cluster-small-group-worker-lc796"}
{"level":"info","ts":"2026-11-03T15:53:25.514Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"a593d80b-56bc-421e-aeeb-3d10cb3696ad","workerReplicas":0,"NumOfHosts":1,"runningPods":0,"diff":0}
{"level":"info","ts":"2026-11-03T15:53:25.514Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"a593d80b-56bc-421e-aeeb-3d10cb3696ad","group":"small-group","indices":{}}
{"level":"info","ts":"2026-11-03T15:53:25.514Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"a593d80b-56bc-421e-aeeb-3d10cb3696ad","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:53:25.514Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"a593d80b-56bc-421e-aeeb-3d10cb3696ad","filter labels":["test-ns-zjpgs",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:53:25.514Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"a593d80b-56bc-421e-aeeb-3d10cb3696ad","filter labels":["test-ns-zjpgs",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:53:25.514Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"a593d80b-56bc-421e-aeeb-3d10cb3696ad","filter labels":["test-ns-zjpgs",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:53:25.514Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"a593d80b-56bc-421e-aeeb-3d10cb3696ad","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:53:24Z","stateTransitionTimes":{"ready":"2026-11-03T15:48:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"ray-cluster-head-jhq58","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-11-03T15:46:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:53:24Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:48:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":5},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:53:25Z","stateTransitionTimes":{"ready":"2026-11-03T15:48:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-11-03T15:46:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:53:24Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:48:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":5}}
{"level":"info","ts":"2026-11-03T15:53:25.576Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"584fa71b-9474-4aa5-860a-e811d8649e41"}
{"level":"info","ts":"2026-11-03T15:53:25.576Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"584fa71b-9474-4aa5-860a-e811d8649e41"}
{"level":"info","ts":"2026-11-03T15:53:25.576Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"584fa71b-9474-4aa5-860a-e811d8649e41","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:53:25.576Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"584fa71b-9474-4aa5-860a-e811d8649e41"}
{"level":"info","ts":"2026-11-03T15:53:25.576Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"f3d210b2-4ddc-42e9-a01c-dc91f6c87c3b","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:53:25.627Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"584fa71b-9474-4aa5-860a-e811d8649e41","error":"certificate ray-head-cert-ray-cluster is not ready at current generation 3"}
{"level":"error","ts":"2026-11-03T15:53:25.627Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"584fa71b-9474-4aa5-860a-e811d8649e41","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 3","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:53:25.627Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"584fa71b-9474-4aa5-860a-e811d8649e41","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 3"}
{"level":"info","ts":"2026-11-03T15:53:25.627Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"584fa71b-9474-4aa5-860a-e811d8649e41","filter labels":["test-ns-zjpgs",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:53:25.627Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"584fa71b-9474-4aa5-860a-e811d8649e41","filter labels":["test-ns-zjpgs",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:53:25.627Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"584fa71b-9474-4aa5-860a-e811d8649e41","filter labels":["test-ns-zjpgs",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:53:25.627Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"584fa71b-9474-4aa5-860a-e811d8649e41","name":"ray-cluster","old status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:53:24Z","stateTransitionTimes":{"ready":"2026-11-03T15:48:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.132.0.54","serviceIP":"10.132.0.54","podName":"ray-cluster-head-jhq58","serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-11-03T15:46:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:53:24Z","reason":"PodFailed","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:48:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":5},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"2G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:53:25Z","stateTransitionTimes":{"ready":"2026-11-03T15:48:44Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"serviceName":"ray-cluster-head-svc"},"conditions":[{"type":"AuthenticationReady","status":"True","observedGeneration":5,"lastTransitionTime":"2026-11-03T15:46:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"},{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:53:24Z","reason":"HeadPodNotFound","message":"Head Pod not found"},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:48:44Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:46:51Z","reason":"RayClusterSuspending","message":""}],"maxWorkerReplicas":3,"observedGeneration":5}}
{"level":"info","ts":"2026-11-03T15:53:25.682Z","logger":"controllers.RayCluster","msg":"Error updating status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"584fa71b-9474-4aa5-860a-e811d8649e41","name":"ray-cluster","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","RayCluster":{"apiVersion":"ray.io/v1","kind":"RayCluster","namespace":"test-ns-zjpgs","name":"ray-cluster"}}
{"level":"info","ts":"2026-11-03T15:53:25.682Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"584fa71b-9474-4aa5-860a-e811d8649e41"}
{"level":"error","ts":"2026-11-03T15:53:25.682Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"584fa71b-9474-4aa5-860a-e811d8649e41","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 3","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:53:25.687Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"30e4407a-ae29-419b-9cb6-9d4d257168bb"}
{"level":"info","ts":"2026-11-03T15:53:25.687Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"30e4407a-ae29-419b-9cb6-9d4d257168bb"}
{"level":"info","ts":"2026-11-03T15:53:25.687Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"30e4407a-ae29-419b-9cb6-9d4d257168bb","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:53:25.688Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"30e4407a-ae29-419b-9cb6-9d4d257168bb"}
{"level":"info","ts":"2026-11-03T15:53:25.740Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"30e4407a-ae29-419b-9cb6-9d4d257168bb","error":"certificate ray-head-cert-ray-cluster is not ready at current generation 3"}
{"level":"error","ts":"2026-11-03T15:53:25.740Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"30e4407a-ae29-419b-9cb6-9d4d257168bb","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 3","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:53:25.740Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"30e4407a-ae29-419b-9cb6-9d4d257168bb","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 3"}
{"level":"info","ts":"2026-11-03T15:53:25.740Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"30e4407a-ae29-419b-9cb6-9d4d257168bb","filter labels":["test-ns-zjpgs",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:53:25.740Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"30e4407a-ae29-419b-9cb6-9d4d257168bb","filter labels":["test-ns-zjpgs",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:53:25.740Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"30e4407a-ae29-419b-9cb6-9d4d257168bb","filter labels":["test-ns-zjpgs",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:53:25.740Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"30e4407a-ae29-419b-9cb6-9d4d257168bb"}
{"level":"error","ts":"2026-11-03T15:53:25.740Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"30e4407a-ae29-419b-9cb6-9d4d257168bb","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 3","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:53:25.750Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"15d15275-3737-4511-a08c-af2e790f18df"}
{"level":"info","ts":"2026-11-03T15:53:25.750Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"15d15275-3737-4511-a08c-af2e790f18df"}
{"level":"info","ts":"2026-11-03T15:53:25.750Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"15d15275-3737-4511-a08c-af2e790f18df","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:53:25.750Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"15d15275-3737-4511-a08c-af2e790f18df"}
{"level":"info","ts":"2026-11-03T15:53:25.801Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"15d15275-3737-4511-a08c-af2e790f18df","error":"certificate ray-head-cert-ray-cluster is not ready at current generation 3"}
{"level":"error","ts":"2026-11-03T15:53:25.801Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"15d15275-3737-4511-a08c-af2e790f18df","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 3","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:53:25.801Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"15d15275-3737-4511-a08c-af2e790f18df","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 3"}
{"level":"info","ts":"2026-11-03T15:53:25.801Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"15d15275-3737-4511-a08c-af2e790f18df","filter labels":["test-ns-zjpgs",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:53:25.801Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"15d15275-3737-4511-a08c-af2e790f18df","filter labels":["test-ns-zjpgs",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:53:25.801Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"15d15275-3737-4511-a08c-af2e790f18df","filter labels":["test-ns-zjpgs",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:53:25.801Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"15d15275-3737-4511-a08c-af2e790f18df"}
{"level":"error","ts":"2026-11-03T15:53:25.801Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"15d15275-3737-4511-a08c-af2e790f18df","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 3","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:53:25.821Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"bbf6dec4-1189-4200-b425-484185a98159"}
{"level":"info","ts":"2026-11-03T15:53:25.821Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"bbf6dec4-1189-4200-b425-484185a98159"}
{"level":"info","ts":"2026-11-03T15:53:25.821Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"bbf6dec4-1189-4200-b425-484185a98159","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:53:25.821Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"bbf6dec4-1189-4200-b425-484185a98159"}
{"level":"info","ts":"2026-11-03T15:53:25.871Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"bbf6dec4-1189-4200-b425-484185a98159","error":"certificate ray-head-cert-ray-cluster is not ready at current generation 3"}
{"level":"error","ts":"2026-11-03T15:53:25.871Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"bbf6dec4-1189-4200-b425-484185a98159","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 3","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:53:25.872Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"bbf6dec4-1189-4200-b425-484185a98159","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 3"}
{"level":"info","ts":"2026-11-03T15:53:25.872Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"bbf6dec4-1189-4200-b425-484185a98159","filter labels":["test-ns-zjpgs",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:53:25.872Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"bbf6dec4-1189-4200-b425-484185a98159","filter labels":["test-ns-zjpgs",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:53:25.872Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"bbf6dec4-1189-4200-b425-484185a98159","filter labels":["test-ns-zjpgs",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:53:25.872Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"bbf6dec4-1189-4200-b425-484185a98159"}
{"level":"error","ts":"2026-11-03T15:53:25.872Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"bbf6dec4-1189-4200-b425-484185a98159","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 3","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:53:25.913Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"f2025ced-cf34-4b48-94ad-3983204ca36a"}
{"level":"info","ts":"2026-11-03T15:53:25.913Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"f2025ced-cf34-4b48-94ad-3983204ca36a"}
{"level":"info","ts":"2026-11-03T15:53:25.913Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"f2025ced-cf34-4b48-94ad-3983204ca36a","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:53:25.913Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"f2025ced-cf34-4b48-94ad-3983204ca36a"}
{"level":"info","ts":"2026-11-03T15:53:25.963Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"f2025ced-cf34-4b48-94ad-3983204ca36a","error":"certificate ray-head-cert-ray-cluster is not ready at current generation 3"}
{"level":"error","ts":"2026-11-03T15:53:25.963Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"f2025ced-cf34-4b48-94ad-3983204ca36a","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 3","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:53:25.963Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"f2025ced-cf34-4b48-94ad-3983204ca36a","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 3"}
{"level":"info","ts":"2026-11-03T15:53:25.963Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"f2025ced-cf34-4b48-94ad-3983204ca36a","filter labels":["test-ns-zjpgs",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:53:25.963Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"f2025ced-cf34-4b48-94ad-3983204ca36a","filter labels":["test-ns-zjpgs",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:53:25.963Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"f2025ced-cf34-4b48-94ad-3983204ca36a","filter labels":["test-ns-zjpgs",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:53:25.963Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"f2025ced-cf34-4b48-94ad-3983204ca36a"}
{"level":"error","ts":"2026-11-03T15:53:25.963Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"f2025ced-cf34-4b48-94ad-3983204ca36a","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 3","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:53:26.044Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"180996d0-612f-46a1-9847-930103fc7001"}
{"level":"info","ts":"2026-11-03T15:53:26.044Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"180996d0-612f-46a1-9847-930103fc7001"}
{"level":"info","ts":"2026-11-03T15:53:26.044Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"180996d0-612f-46a1-9847-930103fc7001","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:53:26.044Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"180996d0-612f-46a1-9847-930103fc7001"}
{"level":"info","ts":"2026-11-03T15:53:26.094Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"180996d0-612f-46a1-9847-930103fc7001","error":"certificate ray-head-cert-ray-cluster is not ready at current generation 3"}
{"level":"error","ts":"2026-11-03T15:53:26.095Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"180996d0-612f-46a1-9847-930103fc7001","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 3","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:53:26.095Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"180996d0-612f-46a1-9847-930103fc7001","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 3"}
{"level":"info","ts":"2026-11-03T15:53:26.095Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"180996d0-612f-46a1-9847-930103fc7001","filter labels":["test-ns-zjpgs",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:53:26.095Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"180996d0-612f-46a1-9847-930103fc7001","filter labels":["test-ns-zjpgs",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:53:26.095Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"180996d0-612f-46a1-9847-930103fc7001","filter labels":["test-ns-zjpgs",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:53:26.095Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"180996d0-612f-46a1-9847-930103fc7001"}
{"level":"error","ts":"2026-11-03T15:53:26.095Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"180996d0-612f-46a1-9847-930103fc7001","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 3","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:53:26.183Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"f3d210b2-4ddc-42e9-a01c-dc91f6c87c3b","certificate":"ray-head-cert-ray-cluster"}
{"level":"info","ts":"2026-11-03T15:53:26.183Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"f3d210b2-4ddc-42e9-a01c-dc91f6c87c3b"}
{"level":"error","ts":"2026-11-03T15:53:26.232Z","logger":"events","msg":"Server rejected event (will not retry!)","event":{"namespace":"test-ns-zjpgs","name":"ray-cluster.18e493eef8a64ace"},"error":"events.events.k8s.io \"ray-cluster.18e493eef8a64ace\" is forbidden: unable to create new content in namespace test-ns-zjpgs because it is being terminated","stacktrace":"k8s.io/client-go/tools/events.recordEvent\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:270\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).attemptRecording\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:211\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).recordToSink.func1\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:200"}
{"level":"info","ts":"2026-11-03T15:53:26.256Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"baaa9bab-a7f5-43de-8ad5-00841c169e82"}
{"level":"info","ts":"2026-11-03T15:53:26.256Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"baaa9bab-a7f5-43de-8ad5-00841c169e82"}
{"level":"info","ts":"2026-11-03T15:53:26.256Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"baaa9bab-a7f5-43de-8ad5-00841c169e82","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:53:26.256Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"baaa9bab-a7f5-43de-8ad5-00841c169e82"}
{"level":"info","ts":"2026-11-03T15:53:26.307Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"baaa9bab-a7f5-43de-8ad5-00841c169e82","error":"certificate ray-head-cert-ray-cluster is not ready at current generation 3"}
{"level":"error","ts":"2026-11-03T15:53:26.307Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"baaa9bab-a7f5-43de-8ad5-00841c169e82","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 3","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:53:26.307Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"baaa9bab-a7f5-43de-8ad5-00841c169e82","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 3"}
{"level":"info","ts":"2026-11-03T15:53:26.307Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"baaa9bab-a7f5-43de-8ad5-00841c169e82","filter labels":["test-ns-zjpgs",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:53:26.307Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"baaa9bab-a7f5-43de-8ad5-00841c169e82","filter labels":["test-ns-zjpgs",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:53:26.307Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"baaa9bab-a7f5-43de-8ad5-00841c169e82","filter labels":["test-ns-zjpgs",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:53:26.307Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"baaa9bab-a7f5-43de-8ad5-00841c169e82"}
{"level":"error","ts":"2026-11-03T15:53:26.307Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"baaa9bab-a7f5-43de-8ad5-00841c169e82","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 3","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:53:26.409Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"1c2e8332-8057-4e24-9ae6-f56a1e72a912","cluster":"raycluster-auth-token"}
{"level":"info","ts":"2026-11-03T15:53:26.628Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"7e487bc4-2d4c-4321-a462-b35b003d20e2"}
{"level":"info","ts":"2026-11-03T15:53:26.628Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"7e487bc4-2d4c-4321-a462-b35b003d20e2"}
{"level":"info","ts":"2026-11-03T15:53:26.628Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"7e487bc4-2d4c-4321-a462-b35b003d20e2","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:53:26.628Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"7e487bc4-2d4c-4321-a462-b35b003d20e2"}
{"level":"info","ts":"2026-11-03T15:53:26.678Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"7e487bc4-2d4c-4321-a462-b35b003d20e2","error":"certificate ray-head-cert-ray-cluster is not ready at current generation 3"}
{"level":"error","ts":"2026-11-03T15:53:26.678Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"7e487bc4-2d4c-4321-a462-b35b003d20e2","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 3","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:53:26.678Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"7e487bc4-2d4c-4321-a462-b35b003d20e2","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 3"}
{"level":"info","ts":"2026-11-03T15:53:26.678Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"7e487bc4-2d4c-4321-a462-b35b003d20e2","filter labels":["test-ns-zjpgs",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:53:26.679Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"7e487bc4-2d4c-4321-a462-b35b003d20e2","filter labels":["test-ns-zjpgs",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:53:26.679Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"7e487bc4-2d4c-4321-a462-b35b003d20e2","filter labels":["test-ns-zjpgs",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:53:26.679Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"7e487bc4-2d4c-4321-a462-b35b003d20e2"}
{"level":"error","ts":"2026-11-03T15:53:26.679Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"7e487bc4-2d4c-4321-a462-b35b003d20e2","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 3","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:53:26.866Z","logger":"controllers.RayClusterMTLS","msg":"Updating worker certificate","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"1c2e8332-8057-4e24-9ae6-f56a1e72a912","certificate":"ray-worker-cert-raycluster-auth-token","ipAddresses":["10.132.0.55","127.0.0.1"]}
{"level":"info","ts":"2026-11-03T15:53:27.176Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificate not ready yet","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"1c2e8332-8057-4e24-9ae6-f56a1e72a912","certificate":"ray-worker-cert-raycluster-auth-token"}
{"level":"info","ts":"2026-11-03T15:53:27.176Z","logger":"controllers.RayClusterMTLS","msg":"TLS certificates not ready yet, will requeue","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"1c2e8332-8057-4e24-9ae6-f56a1e72a912"}
{"level":"info","ts":"2026-11-03T15:53:27.320Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"ab65e34f-81f1-4661-9235-d9a6a880bba1"}
{"level":"info","ts":"2026-11-03T15:53:27.320Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"ab65e34f-81f1-4661-9235-d9a6a880bba1"}
{"level":"info","ts":"2026-11-03T15:53:27.320Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"ab65e34f-81f1-4661-9235-d9a6a880bba1","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:53:27.320Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"ab65e34f-81f1-4661-9235-d9a6a880bba1"}
{"level":"info","ts":"2026-11-03T15:53:27.371Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"ab65e34f-81f1-4661-9235-d9a6a880bba1","error":"certificate ray-head-cert-ray-cluster is not ready at current generation 3"}
{"level":"error","ts":"2026-11-03T15:53:27.371Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"ab65e34f-81f1-4661-9235-d9a6a880bba1","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 3","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:53:27.371Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"ab65e34f-81f1-4661-9235-d9a6a880bba1","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 3"}
{"level":"info","ts":"2026-11-03T15:53:27.371Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"ab65e34f-81f1-4661-9235-d9a6a880bba1","filter labels":["test-ns-zjpgs",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:53:27.371Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"ab65e34f-81f1-4661-9235-d9a6a880bba1","filter labels":["test-ns-zjpgs",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:53:27.371Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"ab65e34f-81f1-4661-9235-d9a6a880bba1","filter labels":["test-ns-zjpgs",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:53:27.371Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"ab65e34f-81f1-4661-9235-d9a6a880bba1"}
{"level":"error","ts":"2026-11-03T15:53:27.371Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"ab65e34f-81f1-4661-9235-d9a6a880bba1","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 3","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:53:28.651Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"ac19d68a-ca2c-43f9-8668-2d1188d35eec"}
{"level":"info","ts":"2026-11-03T15:53:28.651Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"ac19d68a-ca2c-43f9-8668-2d1188d35eec"}
{"level":"info","ts":"2026-11-03T15:53:28.651Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"ac19d68a-ca2c-43f9-8668-2d1188d35eec","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:53:28.651Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"ac19d68a-ca2c-43f9-8668-2d1188d35eec"}
{"level":"info","ts":"2026-11-03T15:53:28.702Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"ac19d68a-ca2c-43f9-8668-2d1188d35eec","error":"certificate ray-head-cert-ray-cluster is not ready at current generation 3"}
{"level":"error","ts":"2026-11-03T15:53:28.702Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"ac19d68a-ca2c-43f9-8668-2d1188d35eec","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 3","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:53:28.702Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"ac19d68a-ca2c-43f9-8668-2d1188d35eec","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 3"}
{"level":"info","ts":"2026-11-03T15:53:28.702Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"ac19d68a-ca2c-43f9-8668-2d1188d35eec","filter labels":["test-ns-zjpgs",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:53:28.702Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"ac19d68a-ca2c-43f9-8668-2d1188d35eec","filter labels":["test-ns-zjpgs",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:53:28.702Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"ac19d68a-ca2c-43f9-8668-2d1188d35eec","filter labels":["test-ns-zjpgs",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:53:28.702Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"ac19d68a-ca2c-43f9-8668-2d1188d35eec"}
{"level":"error","ts":"2026-11-03T15:53:28.702Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"ac19d68a-ca2c-43f9-8668-2d1188d35eec","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 3","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:53:31.263Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"075b56dd-8c3d-490e-aea3-956695fefb88"}
{"level":"info","ts":"2026-11-03T15:53:31.263Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"075b56dd-8c3d-490e-aea3-956695fefb88"}
{"level":"info","ts":"2026-11-03T15:53:31.263Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"075b56dd-8c3d-490e-aea3-956695fefb88","1 head service found":"ray-cluster-head-svc"}
{"level":"info","ts":"2026-11-03T15:53:31.263Z","logger":"controllers.RayCluster","msg":"reconcilePods: Found 0 head Pods; creating a head Pod for the RayCluster.","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"075b56dd-8c3d-490e-aea3-956695fefb88"}
{"level":"info","ts":"2026-11-03T15:53:31.314Z","logger":"controllers.RayCluster","msg":"mTLS secrets not ready, requeuing before head pod creation","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"075b56dd-8c3d-490e-aea3-956695fefb88","error":"certificate ray-head-cert-ray-cluster is not ready at current generation 3"}
{"level":"error","ts":"2026-11-03T15:53:31.314Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"075b56dd-8c3d-490e-aea3-956695fefb88","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcilePods-fm","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 3","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:53:31.314Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"075b56dd-8c3d-490e-aea3-956695fefb88","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 3"}
{"level":"info","ts":"2026-11-03T15:53:31.314Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"075b56dd-8c3d-490e-aea3-956695fefb88","filter labels":["test-ns-zjpgs",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:53:31.314Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"075b56dd-8c3d-490e-aea3-956695fefb88","filter labels":["test-ns-zjpgs",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:53:31.314Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"075b56dd-8c3d-490e-aea3-956695fefb88","filter labels":["test-ns-zjpgs",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:53:31.314Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"075b56dd-8c3d-490e-aea3-956695fefb88"}
{"level":"error","ts":"2026-11-03T15:53:31.314Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"075b56dd-8c3d-490e-aea3-956695fefb88","error":"mTLS secrets not ready: certificate ray-head-cert-ray-cluster is not ready at current generation 3","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:53:32.369Z","logger":"controllers.NetworkPolicy","msg":"Reconciling NetworkPolicies for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"46b4ee6b-a943-4eb6-b83f-3664f6d3e304","cluster":"ray-cluster","namespace":"test-ns-zjpgs"}
{"level":"error","ts":"2026-11-03T15:53:32.418Z","logger":"controllers.NetworkPolicy","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"46b4ee6b-a943-4eb6-b83f-3664f6d3e304","error":"networkpolicies.networking.k8s.io \"ray-cluster-head\" is forbidden: unable to create new content in namespace test-ns-zjpgs because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:53:32.418Z","logger":"controllers.NetworkPolicy","msg":"Reconciling NetworkPolicies for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"bd376489-463f-4524-9db8-9003708c9088","cluster":"ray-cluster","namespace":"test-ns-zjpgs"}
{"level":"info","ts":"2026-11-03T15:53:32.418Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"namespace":"test-ns-zjpgs","name":"ray-cluster","reconcileID":"cf9e5010-723f-44ab-9e9f-fa5438555128","namespacedName":{"name":"ray-cluster","namespace":"test-ns-zjpgs"}}
{"level":"info","ts":"2026-11-03T15:53:32.418Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"namespace":"test-ns-zjpgs","name":"ray-cluster","reconcileID":"cf9e5010-723f-44ab-9e9f-fa5438555128","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:53:32.418Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"namespace":"test-ns-zjpgs","name":"ray-cluster","reconcileID":"cf9e5010-723f-44ab-9e9f-fa5438555128","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:53:32.418Z","logger":"authentication-controller","msg":"Autoscaler RoleBinding does not exist yet, will retry auth SA patch","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"namespace":"test-ns-zjpgs","name":"ray-cluster","reconcileID":"cf9e5010-723f-44ab-9e9f-fa5438555128","cluster":"ray-cluster"}
{"level":"error","ts":"2026-11-03T15:53:32.466Z","logger":"events","msg":"Server rejected event (will not retry!)","event":{"namespace":"test-ns-zjpgs","name":"ray-cluster.18e493f06c44e702"},"error":"events.events.k8s.io \"ray-cluster.18e493f06c44e702\" is forbidden: unable to create new content in namespace test-ns-zjpgs because it is being terminated","stacktrace":"k8s.io/client-go/tools/events.recordEvent\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:270\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).attemptRecording\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:211\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).recordToSink.func1\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:200"}
{"level":"error","ts":"2026-11-03T15:53:32.467Z","logger":"controllers.NetworkPolicy","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"bd376489-463f-4524-9db8-9003708c9088","error":"networkpolicies.networking.k8s.io \"ray-cluster-head\" is forbidden: unable to create new content in namespace test-ns-zjpgs because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:53:32.477Z","logger":"controllers.NetworkPolicy","msg":"Reconciling NetworkPolicies for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"3fe53c68-a87b-4500-a3ca-74f43e7d69a0","cluster":"ray-cluster","namespace":"test-ns-zjpgs"}
{"level":"info","ts":"2026-11-03T15:53:32.479Z","logger":"authentication-controller","msg":"Autoscaler RoleBinding not ready yet, requeueing","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"namespace":"test-ns-zjpgs","name":"ray-cluster","reconcileID":"cf9e5010-723f-44ab-9e9f-fa5438555128","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:53:32.479Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"namespace":"test-ns-zjpgs","name":"ray-cluster","reconcileID":"178485d2-2c98-410c-af94-4d22abb44c90","namespacedName":{"name":"ray-cluster","namespace":"test-ns-zjpgs"}}
{"level":"info","ts":"2026-11-03T15:53:32.479Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"namespace":"test-ns-zjpgs","name":"ray-cluster","reconcileID":"178485d2-2c98-410c-af94-4d22abb44c90","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:53:32.479Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"namespace":"test-ns-zjpgs","name":"ray-cluster","reconcileID":"178485d2-2c98-410c-af94-4d22abb44c90","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:53:32.479Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"49eccab5-a8a4-41c7-97db-a8584fd2853d","cluster":"ray-cluster"}
{"level":"error","ts":"2026-11-03T15:53:32.514Z","logger":"controllers.RayCluster","msg":"Error reconcile resources","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"236902e7-d7b6-4e00-9d5f-bc67e761dbca","function name":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).reconcileAutoscalerServiceAccount-fm","error":"serviceaccounts \"ray-cluster\" is forbidden: unable to create new content in namespace test-ns-zjpgs because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).rayClusterReconcile\n\t/workspace/controllers/ray/raycluster_controller.go:374\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterReconciler).Reconcile\n\t/workspace/controllers/ray/raycluster_controller.go:129\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:53:32.514Z","logger":"controllers.RayCluster","msg":"Reconciliation error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"236902e7-d7b6-4e00-9d5f-bc67e761dbca","error":"serviceaccounts \"ray-cluster\" is forbidden: unable to create new content in namespace test-ns-zjpgs because it is being terminated"}
{"level":"info","ts":"2026-11-03T15:53:32.514Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"236902e7-d7b6-4e00-9d5f-bc67e761dbca","filter labels":["test-ns-zjpgs",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:53:32.514Z","logger":"controllers.RayCluster","msg":"updateEndpoints: Unable to find a Service for this RayCluster. Not adding RayCluster status.endpoints","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"236902e7-d7b6-4e00-9d5f-bc67e761dbca","serviceSelectors":["test-ns-zjpgs",{"ray.io/cluster":"ray-cluster","ray.io/identifier":"ray-cluster-head","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:53:32.514Z","logger":"controllers.RayCluster","msg":"Found 0 head pod","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"236902e7-d7b6-4e00-9d5f-bc67e761dbca","filter labels":["test-ns-zjpgs",{"ray.io/cluster":"ray-cluster","ray.io/node-type":"head"}]}
{"level":"info","ts":"2026-11-03T15:53:32.514Z","logger":"controllers.RayCluster","msg":"Got error when calculating new status","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"236902e7-d7b6-4e00-9d5f-bc67e761dbca","error":"unable to find head service. cluster name ray-cluster, filter labels [test-ns-zjpgs map[ray.io/cluster:ray-cluster ray.io/identifier:ray-cluster-head ray.io/node-type:head]]"}
{"level":"info","ts":"2026-11-03T15:53:32.514Z","logger":"controllers.RayCluster","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"236902e7-d7b6-4e00-9d5f-bc67e761dbca"}
{"level":"error","ts":"2026-11-03T15:53:32.514Z","logger":"controllers.RayCluster","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"236902e7-d7b6-4e00-9d5f-bc67e761dbca","error":"serviceaccounts \"ray-cluster\" is forbidden: unable to create new content in namespace test-ns-zjpgs because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-03T15:53:32.526Z","logger":"controllers.NetworkPolicy","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"3fe53c68-a87b-4500-a3ca-74f43e7d69a0","error":"networkpolicies.networking.k8s.io \"ray-cluster-head\" is forbidden: unable to create new content in namespace test-ns-zjpgs because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:53:32.546Z","logger":"controllers.NetworkPolicy","msg":"Reconciling NetworkPolicies for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"5f8a9527-258e-41a2-a493-4382aec50bbe","cluster":"ray-cluster","namespace":"test-ns-zjpgs"}
{"level":"error","ts":"2026-11-03T15:53:32.563Z","logger":"events","msg":"Server rejected event (will not retry!)","event":{"namespace":"test-ns-zjpgs","name":"ray-cluster.18e493f07201cc0e"},"error":"events.events.k8s.io \"ray-cluster.18e493f07201cc0e\" is forbidden: unable to create new content in namespace test-ns-zjpgs because it is being terminated","stacktrace":"k8s.io/client-go/tools/events.recordEvent\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:270\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).attemptRecording\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:211\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).recordToSink.func1\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:200"}
{"level":"error","ts":"2026-11-03T15:53:32.564Z","logger":"events","msg":"Server rejected event (will not retry!)","event":{"namespace":"test-ns-zjpgs","name":"ray-cluster.18e493f06c44e702"},"error":"events.events.k8s.io \"ray-cluster.18e493f06c44e702\" is forbidden: unable to create new content in namespace test-ns-zjpgs because it is being terminated","stacktrace":"k8s.io/client-go/tools/events.recordEvent\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:270\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).attemptRecording\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:211\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).recordToSink.func1\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:200"}
{"level":"error","ts":"2026-11-03T15:53:32.588Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"namespace":"test-ns-zjpgs","name":"ray-cluster","reconcileID":"178485d2-2c98-410c-af94-4d22abb44c90","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"ray-cluster-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-zjpgs because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:167\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:53:32.588Z","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"namespace":"test-ns-zjpgs","name":"ray-cluster","reconcileID":"178485d2-2c98-410c-af94-4d22abb44c90"}
{"level":"error","ts":"2026-11-03T15:53:32.588Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"namespace":"test-ns-zjpgs","name":"ray-cluster","reconcileID":"178485d2-2c98-410c-af94-4d22abb44c90","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"ray-cluster-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-zjpgs because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:53:32.593Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"namespace":"test-ns-zjpgs","name":"ray-cluster","reconcileID":"d4273a7f-3540-45c2-9512-d67729cc8b34","namespacedName":{"name":"ray-cluster","namespace":"test-ns-zjpgs"}}
{"level":"info","ts":"2026-11-03T15:53:32.594Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"namespace":"test-ns-zjpgs","name":"ray-cluster","reconcileID":"d4273a7f-3540-45c2-9512-d67729cc8b34","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:53:32.594Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"namespace":"test-ns-zjpgs","name":"ray-cluster","reconcileID":"d4273a7f-3540-45c2-9512-d67729cc8b34","cluster":"ray-cluster"}
{"level":"error","ts":"2026-11-03T15:53:32.595Z","logger":"controllers.NetworkPolicy","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"5f8a9527-258e-41a2-a493-4382aec50bbe","error":"networkpolicies.networking.k8s.io \"ray-cluster-head\" is forbidden: unable to create new content in namespace test-ns-zjpgs because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-03T15:53:32.628Z","logger":"controllers.RayClusterMTLS","msg":"Failed to reconcile TLS resource","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"49eccab5-a8a4-41c7-97db-a8584fd2853d","resource":"CA certificate","error":"certificates.cert-manager.io \"ray-ca-certificate-ray-cluster\" is forbidden: unable to create new content in namespace test-ns-zjpgs because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).handleAutoGenerate\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:115\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:94\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:53:32.628Z","logger":"controllers.RayClusterMTLS","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"49eccab5-a8a4-41c7-97db-a8584fd2853d"}
{"level":"error","ts":"2026-11-03T15:53:32.628Z","logger":"controllers.RayClusterMTLS","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"49eccab5-a8a4-41c7-97db-a8584fd2853d","error":"certificates.cert-manager.io \"ray-ca-certificate-ray-cluster\" is forbidden: unable to create new content in namespace test-ns-zjpgs because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:53:32.628Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"b24caf5c-f387-4101-9367-7736801f484a","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:53:32.636Z","logger":"controllers.NetworkPolicy","msg":"Reconciling NetworkPolicies for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"14078dc9-4d5f-45b5-a6c9-f62fb2c80252","cluster":"ray-cluster","namespace":"test-ns-zjpgs"}
{"level":"error","ts":"2026-11-03T15:53:32.644Z","logger":"events","msg":"Server rejected event (will not retry!)","event":{"namespace":"test-ns-zjpgs","name":"ray-cluster.18e493f076d9cba1"},"error":"events.events.k8s.io \"ray-cluster.18e493f076d9cba1\" is forbidden: unable to create new content in namespace test-ns-zjpgs because it is being terminated","stacktrace":"k8s.io/client-go/tools/events.recordEvent\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:270\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).attemptRecording\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:211\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).recordToSink.func1\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:200"}
{"level":"error","ts":"2026-11-03T15:53:32.677Z","logger":"events","msg":"Server rejected event (will not retry!)","event":{"namespace":"test-ns-zjpgs","name":"ray-cluster.18e493f078cf287d"},"error":"events.events.k8s.io \"ray-cluster.18e493f078cf287d\" is forbidden: unable to create new content in namespace test-ns-zjpgs because it is being terminated","stacktrace":"k8s.io/client-go/tools/events.recordEvent\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:270\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).attemptRecording\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:211\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).recordToSink.func1\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:200"}
{"level":"error","ts":"2026-11-03T15:53:32.685Z","logger":"controllers.NetworkPolicy","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"14078dc9-4d5f-45b5-a6c9-f62fb2c80252","error":"networkpolicies.networking.k8s.io \"ray-cluster-head\" is forbidden: unable to create new content in namespace test-ns-zjpgs because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-03T15:53:32.700Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"namespace":"test-ns-zjpgs","name":"ray-cluster","reconcileID":"d4273a7f-3540-45c2-9512-d67729cc8b34","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"ray-cluster-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-zjpgs because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:167\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:53:32.700Z","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"namespace":"test-ns-zjpgs","name":"ray-cluster","reconcileID":"d4273a7f-3540-45c2-9512-d67729cc8b34"}
{"level":"error","ts":"2026-11-03T15:53:32.700Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"namespace":"test-ns-zjpgs","name":"ray-cluster","reconcileID":"d4273a7f-3540-45c2-9512-d67729cc8b34","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"ray-cluster-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-zjpgs because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:53:32.710Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"namespace":"test-ns-zjpgs","name":"ray-cluster","reconcileID":"2e71a2df-cb05-4b99-92b2-1d3395f8d516","namespacedName":{"name":"ray-cluster","namespace":"test-ns-zjpgs"}}
{"level":"info","ts":"2026-11-03T15:53:32.710Z","logger":"authentication-controller","msg":"Detected authentication mode","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"namespace":"test-ns-zjpgs","name":"ray-cluster","reconcileID":"2e71a2df-cb05-4b99-92b2-1d3395f8d516","mode":"IntegratedOAuth","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:53:32.710Z","logger":"authentication-controller","msg":"Handling Integrated OAuth with OIDC configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"namespace":"test-ns-zjpgs","name":"ray-cluster","reconcileID":"2e71a2df-cb05-4b99-92b2-1d3395f8d516","cluster":"ray-cluster"}
{"level":"error","ts":"2026-11-03T15:53:32.734Z","logger":"events","msg":"Server rejected event (will not retry!)","event":{"namespace":"test-ns-zjpgs","name":"ray-cluster.18e493f07c3531b1"},"error":"events.events.k8s.io \"ray-cluster.18e493f07c3531b1\" is forbidden: unable to create new content in namespace test-ns-zjpgs because it is being terminated","stacktrace":"k8s.io/client-go/tools/events.recordEvent\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:270\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).attemptRecording\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:211\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).recordToSink.func1\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:200"}
{"level":"info","ts":"2026-11-03T15:53:32.766Z","logger":"controllers.NetworkPolicy","msg":"Reconciling NetworkPolicies for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"27cb58ea-a85f-4118-a366-9c8fb7b12073","cluster":"ray-cluster","namespace":"test-ns-zjpgs"}
{"level":"error","ts":"2026-11-03T15:53:32.776Z","logger":"controllers.RayClusterMTLS","msg":"Failed to reconcile TLS resource","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"b24caf5c-f387-4101-9367-7736801f484a","resource":"CA certificate","error":"certificates.cert-manager.io \"ray-ca-certificate-ray-cluster\" is forbidden: unable to create new content in namespace test-ns-zjpgs because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).handleAutoGenerate\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:115\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:94\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:53:32.776Z","logger":"controllers.RayClusterMTLS","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"b24caf5c-f387-4101-9367-7736801f484a"}
{"level":"error","ts":"2026-11-03T15:53:32.776Z","logger":"controllers.RayClusterMTLS","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"b24caf5c-f387-4101-9367-7736801f484a","error":"certificates.cert-manager.io \"ray-ca-certificate-ray-cluster\" is forbidden: unable to create new content in namespace test-ns-zjpgs because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:53:32.786Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"b35b233b-9b60-4b0d-b3ad-fb038727e8aa","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:53:32.803Z","logger":"controllers.RayCluster","msg":"RayCluster is being deleted, just ignore","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"5d1f3b20-15d5-43d6-9dfc-dc9390f6bbdf"}
{"level":"error","ts":"2026-11-03T15:53:32.812Z","logger":"authentication-controller","msg":"Failed to update AuthenticationReady condition","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"namespace":"test-ns-zjpgs","name":"ray-cluster","reconcileID":"2e71a2df-cb05-4b99-92b2-1d3395f8d516","error":"Operation cannot be fulfilled on rayclusters.ray.io \"ray-cluster\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).handleOIDCConfiguration\n\t/workspace/controllers/ray/authentication_controller.go:280\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:155\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-03T15:53:32.813Z","logger":"authentication-controller","msg":"Failed to handle authentication configuration","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"namespace":"test-ns-zjpgs","name":"ray-cluster","reconcileID":"2e71a2df-cb05-4b99-92b2-1d3395f8d516","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"ray-cluster-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-zjpgs because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*AuthenticationController).Reconcile\n\t/workspace/controllers/ray/authentication_controller.go:167\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:53:32.813Z","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"namespace":"test-ns-zjpgs","name":"ray-cluster","reconcileID":"2e71a2df-cb05-4b99-92b2-1d3395f8d516"}
{"level":"error","ts":"2026-11-03T15:53:32.813Z","msg":"Reconciler error","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"namespace":"test-ns-zjpgs","name":"ray-cluster","reconcileID":"2e71a2df-cb05-4b99-92b2-1d3395f8d516","error":"failed to ensure OIDC resources: failed to ensure service account: failed to create or update service account: serviceaccounts \"ray-cluster-oauth-proxy-sa\" is forbidden: unable to create new content in namespace test-ns-zjpgs because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:53:32.813Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"namespace":"test-ns-zjpgs","name":"ray-cluster","reconcileID":"ddd4f5f1-f947-4c9e-a2b8-458bb37edc69","namespacedName":{"name":"ray-cluster","namespace":"test-ns-zjpgs"}}
{"level":"info","ts":"2026-11-03T15:53:32.813Z","logger":"authentication-controller","msg":"RayCluster is being deleted, running finalizer","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"namespace":"test-ns-zjpgs","name":"ray-cluster","reconcileID":"ddd4f5f1-f947-4c9e-a2b8-458bb37edc69","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:53:32.813Z","logger":"authentication-controller","msg":"Running authentication resource cleanup (finalizer)","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"namespace":"test-ns-zjpgs","name":"ray-cluster","reconcileID":"ddd4f5f1-f947-4c9e-a2b8-458bb37edc69","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:53:32.813Z","logger":"authentication-controller","msg":"ReferenceGrant cleanup check","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"namespace":"test-ns-zjpgs","name":"ray-cluster","reconcileID":"ddd4f5f1-f947-4c9e-a2b8-458bb37edc69","namespace":"test-ns-zjpgs","otherAuthClusters":0,"action":"will delete"}
{"level":"info","ts":"2026-11-03T15:53:32.813Z","logger":"authentication-controller","msg":"ReferenceGrant already deleted","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"namespace":"test-ns-zjpgs","name":"ray-cluster","reconcileID":"ddd4f5f1-f947-4c9e-a2b8-458bb37edc69","name":"kuberay-gateway-access"}
{"level":"error","ts":"2026-11-03T15:53:32.814Z","logger":"controllers.NetworkPolicy","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"27cb58ea-a85f-4118-a366-9c8fb7b12073","error":"networkpolicies.networking.k8s.io \"ray-cluster-head\" is forbidden: unable to create new content in namespace test-ns-zjpgs because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-03T15:53:32.825Z","logger":"events","msg":"Server rejected event (will not retry!)","event":{"namespace":"test-ns-zjpgs","name":"ray-cluster.18e493f081a0a7ab"},"error":"events.events.k8s.io \"ray-cluster.18e493f081a0a7ab\" is forbidden: unable to create new content in namespace test-ns-zjpgs because it is being terminated","stacktrace":"k8s.io/client-go/tools/events.recordEvent\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:270\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).attemptRecording\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:211\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).recordToSink.func1\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:200"}
{"level":"info","ts":"2026-11-03T15:53:32.865Z","logger":"authentication-controller","msg":"Deleted HTTPRoute","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"namespace":"test-ns-zjpgs","name":"ray-cluster","reconcileID":"ddd4f5f1-f947-4c9e-a2b8-458bb37edc69","name":"test-ns-zjpgs-ray-cluster","namespace":"opendatahub"}
{"level":"error","ts":"2026-11-03T15:53:32.884Z","logger":"controllers.RayClusterMTLS","msg":"Failed to reconcile TLS resource","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"b35b233b-9b60-4b0d-b3ad-fb038727e8aa","resource":"self-signed issuer","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-ray-cluster\" is forbidden: unable to create new content in namespace test-ns-zjpgs because it is being terminated","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).handleAutoGenerate\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:115\ngithub.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayClusterMTLSController).Reconcile\n\t/workspace/controllers/ray/raycluster_mtls_controller.go:94\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:221\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:478\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"info","ts":"2026-11-03T15:53:32.884Z","logger":"controllers.RayClusterMTLS","msg":"Warning: Reconciler returned both a result with either RequeueAfter or Requeue set and a non-nil error. RequeueAfter and Requeue will always be ignored if the error is non-nil. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"b35b233b-9b60-4b0d-b3ad-fb038727e8aa"}
{"level":"error","ts":"2026-11-03T15:53:32.884Z","logger":"controllers.RayClusterMTLS","msg":"Reconciler error","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"reconcileID":"b35b233b-9b60-4b0d-b3ad-fb038727e8aa","error":"issuers.cert-manager.io \"ray-selfsigned-issuer-ray-cluster\" is forbidden: unable to create new content in namespace test-ns-zjpgs because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:494\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:437\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/internal/controller/controller.go:312"}
{"level":"error","ts":"2026-11-03T15:53:32.912Z","logger":"events","msg":"Server rejected event (will not retry!)","event":{"namespace":"test-ns-zjpgs","name":"ray-cluster.18e493f07c3531b1"},"error":"events.events.k8s.io \"ray-cluster.18e493f07c3531b1\" is forbidden: unable to create new content in namespace test-ns-zjpgs because it is being terminated","stacktrace":"k8s.io/client-go/tools/events.recordEvent\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:270\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).attemptRecording\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:211\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).recordToSink.func1\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:200"}
{"level":"info","ts":"2026-11-03T15:53:32.925Z","logger":"raycluster-resource","msg":"default","name":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:53:32.925Z","logger":"raycluster-resource","msg":"setting OpenShift RayCluster defaults","name":"ray-cluster","namespace":"test-ns-zjpgs"}
{"level":"info","ts":"2026-11-03T15:53:32.981Z","logger":"raycluster-resource","msg":"validate update","name":"ray-cluster"}
{"level":"error","ts":"2026-11-03T15:53:32.983Z","logger":"events","msg":"Server rejected event (will not retry!)","event":{"namespace":"test-ns-zjpgs","name":"ray-cluster.18e493f081a0a7ab"},"error":"events.events.k8s.io \"ray-cluster.18e493f081a0a7ab\" is forbidden: unable to create new content in namespace test-ns-zjpgs because it is being terminated","stacktrace":"k8s.io/client-go/tools/events.recordEvent\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:270\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).attemptRecording\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:211\nk8s.io/client-go/tools/events.(*eventBroadcasterImpl).recordToSink.func1\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.0/tools/events/event_broadcaster.go:200"}
{"level":"info","ts":"2026-11-03T15:53:33.036Z","logger":"authentication-controller","msg":"Authentication cleanup complete, finalizer removed","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"namespace":"test-ns-zjpgs","name":"ray-cluster","reconcileID":"ddd4f5f1-f947-4c9e-a2b8-458bb37edc69","cluster":"ray-cluster"}
{"level":"info","ts":"2026-11-03T15:53:33.036Z","logger":"authentication-controller","msg":"Reconciling Authentication","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"namespace":"test-ns-zjpgs","name":"ray-cluster","reconcileID":"0f799103-203a-48aa-b31a-d8757a333713","namespacedName":{"name":"ray-cluster","namespace":"test-ns-zjpgs"}}
{"level":"info","ts":"2026-11-03T15:53:33.036Z","logger":"authentication-controller","msg":"RayCluster not found, likely deleted - checking for orphaned resources","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"namespace":"test-ns-zjpgs","name":"ray-cluster","reconcileID":"0f799103-203a-48aa-b31a-d8757a333713"}
{"level":"info","ts":"2026-11-03T15:53:33.036Z","logger":"authentication-controller","msg":"Checking for orphaned ReferenceGrant","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"namespace":"test-ns-zjpgs","name":"ray-cluster","reconcileID":"0f799103-203a-48aa-b31a-d8757a333713","namespace":"test-ns-zjpgs","authClusters":0}
{"level":"info","ts":"2026-11-03T15:53:33.036Z","logger":"authentication-controller","msg":"No orphaned ReferenceGrant found","controller":"authentication","controllerGroup":"ray.io","controllerKind":"RayCluster","RayCluster":{"name":"ray-cluster","namespace":"test-ns-zjpgs"},"namespace":"test-ns-zjpgs","name":"ray-cluster","reconcileID":"0f799103-203a-48aa-b31a-d8757a333713","namespace":"test-ns-zjpgs"}
{"level":"info","ts":"2026-11-03T15:53:57.177Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"807e0c1d-21fd-4120-87a4-8b1149ef8039","cluster":"raycluster-auth-token"}
{"level":"info","ts":"2026-11-03T15:54:05.623Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"04075187-a390-4a16-a252-a2135fff5833"}
{"level":"info","ts":"2026-11-03T15:54:05.623Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"04075187-a390-4a16-a252-a2135fff5833"}
{"level":"info","ts":"2026-11-03T15:54:05.623Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"04075187-a390-4a16-a252-a2135fff5833","1 head service found":"raycluster-auth-token-head-svc"}
{"level":"info","ts":"2026-11-03T15:54:05.623Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"04075187-a390-4a16-a252-a2135fff5833","Found 1 head Pod":"raycluster-auth-token-head-985vp","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:54:05.623Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"04075187-a390-4a16-a252-a2135fff5833","head Pod":"raycluster-auth-token-head-985vp","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-token-head-985vp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:54:05.623Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"04075187-a390-4a16-a252-a2135fff5833","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:54:05.623Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"04075187-a390-4a16-a252-a2135fff5833","worker Pod":"raycluster-auth-token-small-group-worker-jsd4f","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-token-small-group-worker-jsd4f. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:54:05.623Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"04075187-a390-4a16-a252-a2135fff5833","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:54:05.623Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"04075187-a390-4a16-a252-a2135fff5833","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:54:05.623Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"04075187-a390-4a16-a252-a2135fff5833","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:54:05.623Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"04075187-a390-4a16-a252-a2135fff5833","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:54:05.623Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"04075187-a390-4a16-a252-a2135fff5833","podName":"raycluster-auth-token-head-985vp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:54:05.623Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"04075187-a390-4a16-a252-a2135fff5833","podName":"raycluster-auth-token-head-985vp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:54:05.623Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"04075187-a390-4a16-a252-a2135fff5833","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:54:05.623Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"04075187-a390-4a16-a252-a2135fff5833","seconds":300}
{"level":"info","ts":"2026-11-03T15:54:06.581Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"8ef5fe3c-2742-4b71-b154-84190b556ca9"}
{"level":"info","ts":"2026-11-03T15:54:06.581Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"8ef5fe3c-2742-4b71-b154-84190b556ca9"}
{"level":"info","ts":"2026-11-03T15:54:06.581Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"8ef5fe3c-2742-4b71-b154-84190b556ca9","1 head service found":"raycluster-auth-token-head-svc"}
{"level":"info","ts":"2026-11-03T15:54:06.581Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"8ef5fe3c-2742-4b71-b154-84190b556ca9","Found 1 head Pod":"raycluster-auth-token-head-985vp","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:54:06.581Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"8ef5fe3c-2742-4b71-b154-84190b556ca9","head Pod":"raycluster-auth-token-head-985vp","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-token-head-985vp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:54:06.581Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"8ef5fe3c-2742-4b71-b154-84190b556ca9","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:54:06.582Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"8ef5fe3c-2742-4b71-b154-84190b556ca9","worker Pod":"raycluster-auth-token-small-group-worker-jsd4f","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-token-small-group-worker-jsd4f. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:54:06.582Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"8ef5fe3c-2742-4b71-b154-84190b556ca9","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:54:06.582Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"8ef5fe3c-2742-4b71-b154-84190b556ca9","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:54:06.582Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"8ef5fe3c-2742-4b71-b154-84190b556ca9","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:54:06.582Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"8ef5fe3c-2742-4b71-b154-84190b556ca9","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:54:06.582Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"8ef5fe3c-2742-4b71-b154-84190b556ca9","podName":"raycluster-auth-token-head-985vp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:54:06.582Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"8ef5fe3c-2742-4b71-b154-84190b556ca9","podName":"raycluster-auth-token-head-985vp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:54:06.582Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"8ef5fe3c-2742-4b71-b154-84190b556ca9","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:54:06.582Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"8ef5fe3c-2742-4b71-b154-84190b556ca9","seconds":300}
{"level":"info","ts":"2026-11-03T15:54:10.989Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"ca468466-6879-4cef-8be7-1f3c96f406a9"}
{"level":"info","ts":"2026-11-03T15:54:10.989Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"ca468466-6879-4cef-8be7-1f3c96f406a9"}
{"level":"info","ts":"2026-11-03T15:54:10.989Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"ca468466-6879-4cef-8be7-1f3c96f406a9","1 head service found":"raycluster-auth-token-head-svc"}
{"level":"info","ts":"2026-11-03T15:54:10.989Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"ca468466-6879-4cef-8be7-1f3c96f406a9","Found 1 head Pod":"raycluster-auth-token-head-985vp","Pod status":"Pending","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:54:10.989Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"ca468466-6879-4cef-8be7-1f3c96f406a9","head Pod":"raycluster-auth-token-head-985vp","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-token-head-985vp. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:54:10.990Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"ca468466-6879-4cef-8be7-1f3c96f406a9","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:54:10.990Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"ca468466-6879-4cef-8be7-1f3c96f406a9","worker Pod":"raycluster-auth-token-small-group-worker-jsd4f","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-token-small-group-worker-jsd4f. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:54:10.990Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"ca468466-6879-4cef-8be7-1f3c96f406a9","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:54:10.990Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"ca468466-6879-4cef-8be7-1f3c96f406a9","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:54:10.990Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"ca468466-6879-4cef-8be7-1f3c96f406a9","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:54:10.990Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"ca468466-6879-4cef-8be7-1f3c96f406a9","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:54:10.990Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"ca468466-6879-4cef-8be7-1f3c96f406a9","podName":"raycluster-auth-token-head-985vp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:54:10.990Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not running.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"ca468466-6879-4cef-8be7-1f3c96f406a9","podName":"raycluster-auth-token-head-985vp","pod Status.Phase":"Pending"}
{"level":"info","ts":"2026-11-03T15:54:10.990Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"ca468466-6879-4cef-8be7-1f3c96f406a9","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:54:10.990Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"ca468466-6879-4cef-8be7-1f3c96f406a9","seconds":300}
{"level":"info","ts":"2026-11-03T15:54:11.998Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"80152374-aac2-4bb1-81cc-c033c9b5fd78"}
{"level":"info","ts":"2026-11-03T15:54:11.998Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"80152374-aac2-4bb1-81cc-c033c9b5fd78"}
{"level":"info","ts":"2026-11-03T15:54:11.998Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"80152374-aac2-4bb1-81cc-c033c9b5fd78","1 head service found":"raycluster-auth-token-head-svc"}
{"level":"info","ts":"2026-11-03T15:54:11.998Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"80152374-aac2-4bb1-81cc-c033c9b5fd78","Found 1 head Pod":"raycluster-auth-token-head-985vp","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:54:11.998Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"80152374-aac2-4bb1-81cc-c033c9b5fd78","head Pod":"raycluster-auth-token-head-985vp","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-token-head-985vp. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:54:11.998Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"80152374-aac2-4bb1-81cc-c033c9b5fd78","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:54:11.998Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"80152374-aac2-4bb1-81cc-c033c9b5fd78","worker Pod":"raycluster-auth-token-small-group-worker-jsd4f","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-token-small-group-worker-jsd4f. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:54:11.998Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"80152374-aac2-4bb1-81cc-c033c9b5fd78","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:54:11.998Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"80152374-aac2-4bb1-81cc-c033c9b5fd78","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:54:11.998Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"80152374-aac2-4bb1-81cc-c033c9b5fd78","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:54:11.998Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"80152374-aac2-4bb1-81cc-c033c9b5fd78","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:54:11.999Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"80152374-aac2-4bb1-81cc-c033c9b5fd78","podName":"raycluster-auth-token-head-985vp","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:52:54Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:54:11.999Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"80152374-aac2-4bb1-81cc-c033c9b5fd78","podName":"raycluster-auth-token-head-985vp","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:52:54Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:54:11.999Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"80152374-aac2-4bb1-81cc-c033c9b5fd78","name":"raycluster-auth-token","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:52:55Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.59","serviceIP":"10.134.0.59","podName":"raycluster-auth-token-head-985vp","serviceName":"raycluster-auth-token-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:52:51Z","reason":"PodInitializing","message":"containers with unready status: [ray-head kube-rbac-proxy]; kube-rbac-proxy: "},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:52:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:52:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:52:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:52:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:54:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.59","serviceIP":"10.134.0.59","podName":"raycluster-auth-token-head-985vp","serviceName":"raycluster-auth-token-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:52:51Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:52:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:52:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:52:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:52:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:54:12.059Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"7263993c-56ef-4c5f-90af-753e56dd83d4","cluster":"raycluster-auth-token"}
{"level":"info","ts":"2026-11-03T15:54:14.063Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"9938ad92-1736-4cd1-901a-610fd6cb6ec6"}
{"level":"info","ts":"2026-11-03T15:54:14.064Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"9938ad92-1736-4cd1-901a-610fd6cb6ec6"}
{"level":"info","ts":"2026-11-03T15:54:14.064Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"9938ad92-1736-4cd1-901a-610fd6cb6ec6","1 head service found":"raycluster-auth-token-head-svc"}
{"level":"info","ts":"2026-11-03T15:54:14.064Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"9938ad92-1736-4cd1-901a-610fd6cb6ec6","Found 1 head Pod":"raycluster-auth-token-head-985vp","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:54:14.064Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"9938ad92-1736-4cd1-901a-610fd6cb6ec6","head Pod":"raycluster-auth-token-head-985vp","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-token-head-985vp. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:54:14.064Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"9938ad92-1736-4cd1-901a-610fd6cb6ec6","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:54:14.064Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"9938ad92-1736-4cd1-901a-610fd6cb6ec6","worker Pod":"raycluster-auth-token-small-group-worker-jsd4f","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-token-small-group-worker-jsd4f. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:54:14.064Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"9938ad92-1736-4cd1-901a-610fd6cb6ec6","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:54:14.064Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"9938ad92-1736-4cd1-901a-610fd6cb6ec6","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:54:14.064Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"9938ad92-1736-4cd1-901a-610fd6cb6ec6","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:54:14.064Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"9938ad92-1736-4cd1-901a-610fd6cb6ec6","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:54:14.064Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"9938ad92-1736-4cd1-901a-610fd6cb6ec6","podName":"raycluster-auth-token-head-985vp","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:52:54Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:54:14.065Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"9938ad92-1736-4cd1-901a-610fd6cb6ec6","podName":"raycluster-auth-token-head-985vp","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:52:54Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:54:14.065Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"9938ad92-1736-4cd1-901a-610fd6cb6ec6","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:54:14.065Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"9938ad92-1736-4cd1-901a-610fd6cb6ec6","seconds":300}
{"level":"info","ts":"2026-11-03T15:54:16.674Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"04718624-cd6d-4aac-a4d0-5d387ba6a0eb"}
{"level":"info","ts":"2026-11-03T15:54:16.674Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"04718624-cd6d-4aac-a4d0-5d387ba6a0eb"}
{"level":"info","ts":"2026-11-03T15:54:16.674Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"04718624-cd6d-4aac-a4d0-5d387ba6a0eb","1 head service found":"raycluster-auth-token-head-svc"}
{"level":"info","ts":"2026-11-03T15:54:16.674Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"04718624-cd6d-4aac-a4d0-5d387ba6a0eb","Found 1 head Pod":"raycluster-auth-token-head-985vp","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:54:16.675Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"04718624-cd6d-4aac-a4d0-5d387ba6a0eb","head Pod":"raycluster-auth-token-head-985vp","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-token-head-985vp. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:54:16.675Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"04718624-cd6d-4aac-a4d0-5d387ba6a0eb","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:54:16.675Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"04718624-cd6d-4aac-a4d0-5d387ba6a0eb","worker Pod":"raycluster-auth-token-small-group-worker-jsd4f","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-token-small-group-worker-jsd4f. The Pod status is Pending, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:54:16.675Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"04718624-cd6d-4aac-a4d0-5d387ba6a0eb","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:54:16.675Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"04718624-cd6d-4aac-a4d0-5d387ba6a0eb","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:54:16.675Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"04718624-cd6d-4aac-a4d0-5d387ba6a0eb","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:54:16.675Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"04718624-cd6d-4aac-a4d0-5d387ba6a0eb","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:54:16.675Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"04718624-cd6d-4aac-a4d0-5d387ba6a0eb","podName":"raycluster-auth-token-head-985vp","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:52:54Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:54:16.675Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"04718624-cd6d-4aac-a4d0-5d387ba6a0eb","podName":"raycluster-auth-token-head-985vp","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:52:54Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:54:16.675Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"04718624-cd6d-4aac-a4d0-5d387ba6a0eb","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:54:16.675Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"04718624-cd6d-4aac-a4d0-5d387ba6a0eb","seconds":300}
{"level":"info","ts":"2026-11-03T15:54:17.677Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"18ac9669-96e3-42c3-bf83-90a55b5835df"}
{"level":"info","ts":"2026-11-03T15:54:17.677Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"18ac9669-96e3-42c3-bf83-90a55b5835df"}
{"level":"info","ts":"2026-11-03T15:54:17.677Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"18ac9669-96e3-42c3-bf83-90a55b5835df","1 head service found":"raycluster-auth-token-head-svc"}
{"level":"info","ts":"2026-11-03T15:54:17.677Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"18ac9669-96e3-42c3-bf83-90a55b5835df","Found 1 head Pod":"raycluster-auth-token-head-985vp","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:54:17.677Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"18ac9669-96e3-42c3-bf83-90a55b5835df","head Pod":"raycluster-auth-token-head-985vp","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-token-head-985vp. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:54:17.677Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"18ac9669-96e3-42c3-bf83-90a55b5835df","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:54:17.677Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"18ac9669-96e3-42c3-bf83-90a55b5835df","worker Pod":"raycluster-auth-token-small-group-worker-jsd4f","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-token-small-group-worker-jsd4f. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:54:17.677Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"18ac9669-96e3-42c3-bf83-90a55b5835df","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:54:17.677Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"18ac9669-96e3-42c3-bf83-90a55b5835df","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:54:17.677Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"18ac9669-96e3-42c3-bf83-90a55b5835df","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:54:17.677Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"18ac9669-96e3-42c3-bf83-90a55b5835df","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:54:17.678Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"18ac9669-96e3-42c3-bf83-90a55b5835df","podName":"raycluster-auth-token-head-985vp","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:52:54Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:54:17.678Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"18ac9669-96e3-42c3-bf83-90a55b5835df","podName":"raycluster-auth-token-head-985vp","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:52:54Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:54:17.678Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"18ac9669-96e3-42c3-bf83-90a55b5835df","name":"raycluster-auth-token","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:54:11Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.59","serviceIP":"10.134.0.59","podName":"raycluster-auth-token-head-985vp","serviceName":"raycluster-auth-token-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:52:51Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:52:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:52:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:52:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:52:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:54:17Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.59","serviceIP":"10.134.0.59","podName":"raycluster-auth-token-head-985vp","serviceName":"raycluster-auth-token-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:52:51Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:52:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:52:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:52:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:52:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:54:17.739Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"7b102bcc-3eac-4af6-adf4-2e5cbf963f09","cluster":"raycluster-auth-token"}
{"level":"info","ts":"2026-11-03T15:54:19.739Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"5a2114fb-07d2-41de-bc2e-a7697e156e67"}
{"level":"info","ts":"2026-11-03T15:54:19.739Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"5a2114fb-07d2-41de-bc2e-a7697e156e67"}
{"level":"info","ts":"2026-11-03T15:54:19.739Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"5a2114fb-07d2-41de-bc2e-a7697e156e67","1 head service found":"raycluster-auth-token-head-svc"}
{"level":"info","ts":"2026-11-03T15:54:19.739Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"5a2114fb-07d2-41de-bc2e-a7697e156e67","Found 1 head Pod":"raycluster-auth-token-head-985vp","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:54:19.739Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"5a2114fb-07d2-41de-bc2e-a7697e156e67","head Pod":"raycluster-auth-token-head-985vp","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-token-head-985vp. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:54:19.739Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"5a2114fb-07d2-41de-bc2e-a7697e156e67","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:54:19.739Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"5a2114fb-07d2-41de-bc2e-a7697e156e67","worker Pod":"raycluster-auth-token-small-group-worker-jsd4f","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-token-small-group-worker-jsd4f. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:54:19.739Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"5a2114fb-07d2-41de-bc2e-a7697e156e67","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:54:19.739Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"5a2114fb-07d2-41de-bc2e-a7697e156e67","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:54:19.739Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"5a2114fb-07d2-41de-bc2e-a7697e156e67","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:54:19.739Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"5a2114fb-07d2-41de-bc2e-a7697e156e67","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:54:19.739Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"5a2114fb-07d2-41de-bc2e-a7697e156e67","podName":"raycluster-auth-token-head-985vp","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:52:54Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:54:19.740Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"5a2114fb-07d2-41de-bc2e-a7697e156e67","podName":"raycluster-auth-token-head-985vp","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:52:54Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"}}
{"level":"info","ts":"2026-11-03T15:54:19.740Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"5a2114fb-07d2-41de-bc2e-a7697e156e67","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:54:19.740Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"5a2114fb-07d2-41de-bc2e-a7697e156e67","seconds":300}
{"level":"info","ts":"2026-11-03T15:54:23.032Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"56dc7879-733d-48b6-afe9-58ac80c2a803"}
{"level":"info","ts":"2026-11-03T15:54:23.032Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"56dc7879-733d-48b6-afe9-58ac80c2a803"}
{"level":"info","ts":"2026-11-03T15:54:23.032Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"56dc7879-733d-48b6-afe9-58ac80c2a803","1 head service found":"raycluster-auth-token-head-svc"}
{"level":"info","ts":"2026-11-03T15:54:23.032Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"56dc7879-733d-48b6-afe9-58ac80c2a803","Found 1 head Pod":"raycluster-auth-token-head-985vp","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:54:23.032Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"56dc7879-733d-48b6-afe9-58ac80c2a803","head Pod":"raycluster-auth-token-head-985vp","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-token-head-985vp. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:54:23.032Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"56dc7879-733d-48b6-afe9-58ac80c2a803","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:54:23.032Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"56dc7879-733d-48b6-afe9-58ac80c2a803","worker Pod":"raycluster-auth-token-small-group-worker-jsd4f","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-token-small-group-worker-jsd4f. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:54:23.032Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"56dc7879-733d-48b6-afe9-58ac80c2a803","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:54:23.032Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"56dc7879-733d-48b6-afe9-58ac80c2a803","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:54:23.032Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"56dc7879-733d-48b6-afe9-58ac80c2a803","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:54:23.032Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"56dc7879-733d-48b6-afe9-58ac80c2a803","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:54:23.033Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"56dc7879-733d-48b6-afe9-58ac80c2a803","podName":"raycluster-auth-token-small-group-worker-jsd4f","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:52:54Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:54:23.033Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"56dc7879-733d-48b6-afe9-58ac80c2a803","podName":"raycluster-auth-token-small-group-worker-jsd4f","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:52:54Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:54:23.033Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"56dc7879-733d-48b6-afe9-58ac80c2a803","name":"raycluster-auth-token","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:54:17Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.59","serviceIP":"10.134.0.59","podName":"raycluster-auth-token-head-985vp","serviceName":"raycluster-auth-token-head-svc"},"conditions":[{"type":"HeadPodReady","status":"False","lastTransitionTime":"2026-11-03T15:52:51Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-head]"},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:52:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:52:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:52:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:52:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:54:23Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.59","serviceIP":"10.134.0.59","podName":"raycluster-auth-token-head-985vp","serviceName":"raycluster-auth-token-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:54:23Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:52:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:52:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:52:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:52:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:54:23.093Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"6736bec6-a8ed-4ff2-a50e-c12014453c31","cluster":"raycluster-auth-token"}
{"level":"info","ts":"2026-11-03T15:54:25.094Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"afaa324e-8e7f-4a9f-9072-3f6899ca7ad8"}
{"level":"info","ts":"2026-11-03T15:54:25.094Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"afaa324e-8e7f-4a9f-9072-3f6899ca7ad8"}
{"level":"info","ts":"2026-11-03T15:54:25.094Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"afaa324e-8e7f-4a9f-9072-3f6899ca7ad8","1 head service found":"raycluster-auth-token-head-svc"}
{"level":"info","ts":"2026-11-03T15:54:25.094Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"afaa324e-8e7f-4a9f-9072-3f6899ca7ad8","Found 1 head Pod":"raycluster-auth-token-head-985vp","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:54:25.094Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"afaa324e-8e7f-4a9f-9072-3f6899ca7ad8","head Pod":"raycluster-auth-token-head-985vp","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-token-head-985vp. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:54:25.094Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"afaa324e-8e7f-4a9f-9072-3f6899ca7ad8","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:54:25.094Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"afaa324e-8e7f-4a9f-9072-3f6899ca7ad8","worker Pod":"raycluster-auth-token-small-group-worker-jsd4f","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-token-small-group-worker-jsd4f. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:54:25.094Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"afaa324e-8e7f-4a9f-9072-3f6899ca7ad8","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:54:25.094Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"afaa324e-8e7f-4a9f-9072-3f6899ca7ad8","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:54:25.094Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"afaa324e-8e7f-4a9f-9072-3f6899ca7ad8","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:54:25.094Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"afaa324e-8e7f-4a9f-9072-3f6899ca7ad8","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:54:25.094Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"afaa324e-8e7f-4a9f-9072-3f6899ca7ad8","podName":"raycluster-auth-token-small-group-worker-jsd4f","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:52:54Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:54:25.094Z","logger":"controllers.RayCluster","msg":"CheckAllPodsRunning: Pod is not ready.","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"afaa324e-8e7f-4a9f-9072-3f6899ca7ad8","podName":"raycluster-auth-token-small-group-worker-jsd4f","pod Status.Conditions[PodReady]":{"type":"Ready","observedGeneration":1,"status":"False","lastProbeTime":null,"lastTransitionTime":"2026-11-03T15:52:54Z","reason":"ContainersNotReady","message":"containers with unready status: [ray-worker]"}}
{"level":"info","ts":"2026-11-03T15:54:25.094Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"afaa324e-8e7f-4a9f-9072-3f6899ca7ad8","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:54:25.094Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"afaa324e-8e7f-4a9f-9072-3f6899ca7ad8","seconds":300}
{"level":"info","ts":"2026-11-03T15:54:28.699Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"47c6034f-fa5f-4646-9c99-f8200577097c"}
{"level":"info","ts":"2026-11-03T15:54:28.699Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"47c6034f-fa5f-4646-9c99-f8200577097c"}
{"level":"info","ts":"2026-11-03T15:54:28.699Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"47c6034f-fa5f-4646-9c99-f8200577097c","1 head service found":"raycluster-auth-token-head-svc"}
{"level":"info","ts":"2026-11-03T15:54:28.699Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"47c6034f-fa5f-4646-9c99-f8200577097c","Found 1 head Pod":"raycluster-auth-token-head-985vp","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:54:28.699Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"47c6034f-fa5f-4646-9c99-f8200577097c","head Pod":"raycluster-auth-token-head-985vp","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-token-head-985vp. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:54:28.699Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"47c6034f-fa5f-4646-9c99-f8200577097c","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:54:28.699Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"47c6034f-fa5f-4646-9c99-f8200577097c","worker Pod":"raycluster-auth-token-small-group-worker-jsd4f","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-token-small-group-worker-jsd4f. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:54:28.699Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"47c6034f-fa5f-4646-9c99-f8200577097c","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:54:28.699Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"47c6034f-fa5f-4646-9c99-f8200577097c","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:54:28.699Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"47c6034f-fa5f-4646-9c99-f8200577097c","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:54:28.699Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"47c6034f-fa5f-4646-9c99-f8200577097c","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:54:28.699Z","logger":"controllers.RayCluster","msg":"updateRayClusterStatus","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"47c6034f-fa5f-4646-9c99-f8200577097c","name":"raycluster-auth-token","old status":{"desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:54:23Z","endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.59","serviceIP":"10.134.0.59","podName":"raycluster-auth-token-head-985vp","serviceName":"raycluster-auth-token-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:54:23Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"False","lastTransitionTime":"2026-11-03T15:52:51Z","reason":"RayClusterPodsProvisioning","message":"RayCluster Pods are being provisioned for first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:52:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:52:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:52:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1},"new status":{"state":"ready","desiredCPU":"1","desiredMemory":"7G","desiredGPU":"0","desiredTPU":"0","lastUpdateTime":"2026-11-03T15:54:28Z","stateTransitionTimes":{"ready":"2026-11-03T15:54:28Z"},"endpoints":{"client":"10001","dashboard":"8265","gcs-server":"6379","metrics":"8080","serve":"8000"},"head":{"podIP":"10.134.0.59","serviceIP":"10.134.0.59","podName":"raycluster-auth-token-head-985vp","serviceName":"raycluster-auth-token-head-svc"},"conditions":[{"type":"HeadPodReady","status":"True","lastTransitionTime":"2026-11-03T15:54:23Z","reason":"HeadPodRunningAndReady","message":""},{"type":"RayClusterProvisioned","status":"True","lastTransitionTime":"2026-11-03T15:54:28Z","reason":"AllPodRunningAndReadyFirstTime","message":"All Ray Pods are ready for the first time"},{"type":"RayClusterSuspended","status":"False","lastTransitionTime":"2026-11-03T15:52:51Z","reason":"RayClusterSuspended","message":""},{"type":"RayClusterSuspending","status":"False","lastTransitionTime":"2026-11-03T15:52:51Z","reason":"RayClusterSuspending","message":""},{"type":"AuthenticationReady","status":"True","observedGeneration":1,"lastTransitionTime":"2026-11-03T15:52:51Z","reason":"AuthenticationResourcesCreated","message":"Authentication resources created successfully (mode: IntegratedOAuth)"}],"readyWorkerReplicas":1,"availableWorkerReplicas":1,"desiredWorkerReplicas":1,"minWorkerReplicas":1,"maxWorkerReplicas":1,"observedGeneration":1}}
{"level":"info","ts":"2026-11-03T15:54:28.765Z","logger":"controllers.RayClusterMTLS","msg":"Reconciling auto-generated mTLS for RayCluster","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"8c54c99b-2dae-4d8b-a2bd-835a540bb159","cluster":"raycluster-auth-token"}
{"level":"info","ts":"2026-11-03T15:54:30.765Z","logger":"controllers.RayCluster","msg":"Reconciling Ingress","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"cb9a892e-d85a-4cdd-b2e4-69f9aaac5e64"}
{"level":"info","ts":"2026-11-03T15:54:30.765Z","logger":"controllers.RayCluster","msg":"Reconciling Auth","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"cb9a892e-d85a-4cdd-b2e4-69f9aaac5e64"}
{"level":"info","ts":"2026-11-03T15:54:30.765Z","logger":"controllers.RayCluster","msg":"reconcileHeadService","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"cb9a892e-d85a-4cdd-b2e4-69f9aaac5e64","1 head service found":"raycluster-auth-token-head-svc"}
{"level":"info","ts":"2026-11-03T15:54:30.765Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"cb9a892e-d85a-4cdd-b2e4-69f9aaac5e64","Found 1 head Pod":"raycluster-auth-token-head-985vp","Pod status":"Running","Pod status reason":"","Pod restart policy":"Always","Ray container terminated status":"nil"}
{"level":"info","ts":"2026-11-03T15:54:30.765Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"cb9a892e-d85a-4cdd-b2e4-69f9aaac5e64","head Pod":"raycluster-auth-token-head-985vp","shouldDelete":false,"reason":"KubeRay does not need to delete the head Pod raycluster-auth-token-head-985vp. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:54:30.765Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"cb9a892e-d85a-4cdd-b2e4-69f9aaac5e64","desired workerReplicas (always adhering to minReplicas/maxReplica)":1,"worker group":"small-group","maxReplicas":1,"minReplicas":1,"replicas":1}
{"level":"info","ts":"2026-11-03T15:54:30.765Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"cb9a892e-d85a-4cdd-b2e4-69f9aaac5e64","worker Pod":"raycluster-auth-token-small-group-worker-jsd4f","shouldDelete":false,"reason":"KubeRay does not need to delete the worker Pod raycluster-auth-token-small-group-worker-jsd4f. The Pod status is Running, and the Ray container terminated status is nil."}
{"level":"info","ts":"2026-11-03T15:54:30.765Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"cb9a892e-d85a-4cdd-b2e4-69f9aaac5e64","removing the pods in the scaleStrategy of":"small-group"}
{"level":"info","ts":"2026-11-03T15:54:30.765Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"cb9a892e-d85a-4cdd-b2e4-69f9aaac5e64","workerReplicas":1,"NumOfHosts":1,"runningPods":1,"diff":0}
{"level":"info","ts":"2026-11-03T15:54:30.765Z","logger":"controllers.RayCluster","msg":"reconcilePods: found existing replica indices","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"cb9a892e-d85a-4cdd-b2e4-69f9aaac5e64","group":"small-group","indices":{"0":true}}
{"level":"info","ts":"2026-11-03T15:54:30.765Z","logger":"controllers.RayCluster","msg":"reconcilePods","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"cb9a892e-d85a-4cdd-b2e4-69f9aaac5e64","all workers already exist for group":"small-group"}
{"level":"info","ts":"2026-11-03T15:54:30.766Z","logger":"controllers.RayCluster","msg":"Environment variable is not set, using default value of seconds","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"cb9a892e-d85a-4cdd-b2e4-69f9aaac5e64","environmentVariable":"RAYCLUSTER_DEFAULT_REQUEUE_SECONDS_ENV","defaultValue":300}
{"level":"info","ts":"2026-11-03T15:54:30.766Z","logger":"controllers.RayCluster","msg":"Unconditional requeue after","RayCluster":{"name":"raycluster-auth-token","namespace":"test-ns-rhs44"},"reconcileID":"cb9a892e-d85a-4cdd-b2e4-69f9aaac5e64","seconds":300}